Merge pull request #4905 from WeDataSphere/master-1.1.15-merge

[Feature][Extensions] Add Monitor Server

linkis-dist/bin/install.sh

@@ -127,6 +127,7 @@ RANDOM_LINKISCLI_TOKEN="LINKISCLI-`cat /proc/sys/kernel/random/uuid | awk -F- '{
 if [ $DEBUG_MODE != "true" ];then
   sed -i ${txt}  "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $LINKIS_HOME/conf/linkis-cli/linkis-cli.properties
   sed -i ${txt}  "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $common_conf
+  sed -i ${txt}  "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $LINKIS_HOME/admin/configuration_helper.sh
   sed -i ${txt}  "s#LINKIS_CLI_TEST#$RANDOM_LINKIS_CLI_TEST_TOKEN#g" $common_conf
   sed -i ${txt}  "s#WS-AUTH#$RANDOM_WS_TOKEN#g" $common_conf
   sed -i ${txt}  "s#DSM-AUTH#$RANDOM_DSM_TOKEN#g" $common_conf
@@ -532,6 +533,7 @@ sed -i ${txt}  "s#spring.eureka.instance.metadata-map.linkis.conf.version.*#spri
 if [ "$RESULT_SET_ROOT_PATH" != "" ]
 then
   sed -i ${txt}  "s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g" $entrance_conf
+  sed -i ${txt}  "s#resultSetRootDir=.*#resultSetRootDir=$RESULT_SET_ROOT_PATH#g" $LINKIS_HOME/admin/linkis_task_res_log_clear.sh
 fi
 
 publicservice_conf=$LINKIS_HOME/conf/linkis-ps-publicservice.properties

linkis-dist/package/admin/clear_ec_record.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# description:  clear linkis_ps_job_history_group_history  10 days ago record
+#
+if [ -f ${LINKIS_CONF_DIR}/db.sh ]
+then
+   export LINKIS_DB_CONFIG_PATH=${LINKIS_CONF_DIR}/db.sh
+else
+   if [ -f ${LINKIS_HOME}/conf/db.sh ]
+   then
+      export LINKIS_DB_CONFIG_PATH=${LINKIS_HOME}/conf/db.sh
+   else
+      echo "can not find db.sh"
+      exit
+   fi
+fi
+source ${LINKIS_DB_CONFIG_PATH}
+
+delete_day=`date -d "-10 days"  "+%Y-%m-%d"`
+delte_time="$delete_day 00:00:00"
+echo "start to delete linkis_cg_ec_resource_info_record before $delte_time"
+parm="release_time <=\"$delte_time\" "
+
+count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_cg_ec_resource_info_record where $parm limit 1"`
+maxid=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "SELECT MAX(id) FROM linkis_cg_ec_resource_info_record where $parm limit 1"`
+echo "will delete count:$count"
+echo "maxid:$maxid"
+
+while [ $count -gt 1 ];do
+    mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "DELETE FROM linkis_cg_ec_resource_info_record  where id <= $maxid limit 5000;"
+    count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_cg_ec_resource_info_record where $parm limit 1"`
+    echo "count change ： $count"
+    sleep 1s
+done
+
+echo "clear_ec_record.sh  over"

linkis-dist/package/admin/clear_history_task.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# description:  clear linkis_ps_job_history_group_history  3 month record
+#
+if [ -f ${LINKIS_CONF_DIR}/db.sh ]
+then
+   export LINKIS_DB_CONFIG_PATH=${LINKIS_CONF_DIR}/db.sh
+else
+   if [ -f ${LINKIS_HOME}/conf/db.sh ]
+   then
+      export LINKIS_DB_CONFIG_PATH=${LINKIS_HOME}/conf/db.sh
+   else
+      echo "can not find db.sh"
+      exit
+   fi
+fi
+source ${LINKIS_DB_CONFIG_PATH}
+
+delete_day=`date -d "-90 days"  "+%Y-%m-%d"`
+delte_time="$delete_day 00:00:00"
+echo "start to delete linkis_ps_job_history_group_history before $delte_time"
+parm="created_time <=\"$delte_time\" "
+
+count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_ps_job_history_group_history where $parm limit 1 "`
+maxid=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "SELECT MAX(id) FROM linkis_ps_job_history_group_history where $parm limit 1 "`
+echo "will delete count:$count"
+echo "maxid:$maxid"
+
+while [ $count -gt 1 ];do
+   mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "DELETE FROM linkis_ps_job_history_group_history where id <= $maxid limit 5000;"
+   count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER  -p$MYSQL_PASSWORD  $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_ps_job_history_group_history where $parm limit 1 "`
+   echo "count change ： $count"
+   sleep 1s
+done

linkis-dist/package/admin/configuration_helper.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+cd `dirname $0`
+cd ..
+INSTALL_HOME=`pwd`
+
+
+function print_usage(){
+  echo "Usage: configuration_helper.sh [add | get | delete] [engineType] [version] [creator] [configKey] [configValue option]"
+  echo "get eq: sh  configuration_helper.sh get spark 2.4.3 test wds.linkis.rm.instance hadoop"
+  echo "delete eq: sh  configuration_helper.sh delete spark 2.4.3 test wds.linkis.rm.instance hadoop"
+  echo "add eq: sh  configuration_helper.sh add spark 2.4.3 test wds.linkis.rm.instance hadoop 6"
+  echo "add eq: sh  configuration_helper.sh add spark 2.4.3 test wds.linkis.rm.instance hadoop 6 force"
+  echo "add tips: add with force will ignore check error"
+  echo "Most commands print help when invoked w/o parameters."
+}
+
+if [ $# -lt 6 ]; then
+  print_usage
+  exit 2
+fi
+
+# set LINKIS_HOME
+if [ "$LINKIS_HOME" = "" ]; then
+  export LINKIS_HOME=$INSTALL_HOME
+fi
+
+# set LINKIS_CONF_DIR
+if [ "$LINKIS_CONF_DIR" = "" ]; then
+  export LINKIS_CONF_DIR=$LINKIS_HOME/conf
+fi
+linkisMainConf=$LINKIS_CONF_DIR/linkis.properties
+gatewayUrl=$(grep wds.linkis.gateway.url $linkisMainConf | cut -d"=" -f2)
+echo "gatewayUrl: $gatewayUrl"
+engineType=$2
+version=$3
+creator=$4
+configKey=$5
+user=$6
+configValue=$7
+COMMAND=$1
+if [ "$8" = "force" ]; then
+ force=true
+fi
+
+get()
+{
+  requestUrl="$gatewayUrl/api/rest_j/v1/configuration/keyvalue?creator=$creator&engineType=$engineType&version=$version&configKey=$configKey"
+  curl --location --request GET $requestUrl -H "Token-Code:BML-AUTH" -H "Token-User:$user"
+}
+
+delete()
+{
+  requestUrl="$gatewayUrl/api/rest_j/v1/configuration/keyvalue"
+  requestBody="{\"engineType\":\"$engineType\",\"version\":\"$version\",\"creator\":\"$creator\",\"configKey\":\"$configKey\"}"
+  curl -i -X DELETE $requestUrl  -H "Accept: application/json"  -H "Content-Type: application/json" -H "Token-Code:BML-AUTH" -H "Token-User:$user" -d "$requestBody"
+}
+
+add()
+{
+  requestUrl="$gatewayUrl/api/rest_j/v1/configuration/keyvalue"
+  requestBody="{\"engineType\":\"$engineType\",\"version\":\"$version\",\"creator\":\"$creator\",\"configKey\":\"$configKey\",\"configValue\":\"$configValue\",\"force\":\"$force\",\"user\":\"$user\"}"
+  curl -i -X POST $requestUrl  -H "Accept: application/json"  -H "Content-Type: application/json" -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -d "$requestBody"
+}
+
+case $COMMAND in
+  add|get|delete)
+    $COMMAND
+    ;;
+  *)
+    print_usage
+    exit 2
+    ;;
+esac

linkis-dist/package/admin/linkis_task_res_log_clear.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#!/bin/bash
+expiredDays=365
+resultSetRootDir=/tmp/linkis
+logRootDir=/tmp/linkis
+userResultSetDir=$(hdfs dfs -ls $resultSetRootDir | awk '{print $8}')
+realLogRootDir=$logRootDir/log
+echo userResultSetDirs: $userResultSetDir
+echo realLogRootDir: $realLogRootDir
+
+if [ -z $LINKIS_LOG_DIR ];then
+ expiredFileRecordDir=${LINKIS_HOME}/expiredFileRecord
+else
+ expiredFileRecordDir=$LINKIS_LOG_DIR/expiredFileRecord
+fi
+
+function createExpiredFileRecoredDir(){
+ if [ ! -d $expiredFileRecordDir ];then
+  mkdir -p $expiredFileRecordDir
+ fi
+}
+
+createExpiredFileRecoredDir
+expireDate=$(date -d -${expiredDays}day +%Y-%m-%d)
+expireResultSetFile=$expiredFileRecordDir/linkis_expire_resultset_dir_${expireDate}.txt
+expireLogFile=$expiredFileRecordDir/linkis_expire_log_dir_${expireDate}.txt
+
+hdfs dfs -ls $realLogRootDir | awk '$8 ~ /.*linkis\/log\/[0-9|\-|\_]*/ {cmd = "date -d -12month +%Y-%m-%d";cmd | getline oneMonthAgo;if($6 < oneMonthAgo) print $8}' >> $expireLogFile
+
+for i in $userResultSetDir
+do
+  hdfs dfs -ls $i/linkis | awk '$8 ~ /.*linkis\/[0-9\-]{10}/ {cmd = "date -d -12month +%Y-%m-%d";cmd | getline oneMonthAgo;if($6 < oneMonthAgo) print $8}' >> $expireResultSetFile
+done
+
+cat $expireLogFile | xargs -n 1000 hdfs dfs -rm -r -f
+
+cat $expireResultSetFile | xargs -n 1000 hdfs dfs -rm -r -f
+
+