Skip to content

Commit

Permalink
[gantry][tests] Handler warning better from docker command. Do not us…
Browse files Browse the repository at this point in the history
…e /tmp folder for tests.

Sometimes there could be a problem read files from /tmp.
  • Loading branch information
shizunge committed Nov 30, 2024
1 parent 8b6fe71 commit e003ede
Show file tree
Hide file tree
Showing 15 changed files with 343 additions and 257 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
.shellspec-quick.log
coverage
gantry-test-tmp
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ Labels can be added to services to modify the behavior of *Gantry* for particula
| `gantry.update.options=<string>` | Override [`GANTRY_UPDATE_OPTIONS`](#to-add-options-to-services-update). |
| `gantry.update.timeout_seconds=<number>` | Override [`GANTRY_UPDATE_TIMEOUT_SECONDS`](#to-add-options-to-services-update). |

> NOTE: You must apply the labels to the services not the containers. If you are using docker compose files to setup your services, you need to add the label to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section.
> NOTE: You must apply the labels on the services not on the containers. If you are using docker compose files to setup your services, you need to add the label to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section.
## FAQ

Expand Down
6 changes: 3 additions & 3 deletions src/docker_hub_rate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#

_curl_installed() {
curl --version 1>/dev/null 2>&1;
curl --version 1>/dev/null 2>/dev/null;
}

_docker_hub_rate_token() {
Expand Down Expand Up @@ -66,10 +66,10 @@ _docker_hub_echo_error() {
docker_hub_rate() {
local IMAGE="${1:-ratelimitpreview/test}"
local USER_AND_PASS="${2}"
if ! type log 1>/dev/null 2>&1; then
if ! type log 1>/dev/null 2>/dev/null; then
log() { echo "${*}" >&2; }
fi
if ! type log_lines 1>/dev/null 2>&1; then
if ! type log_lines 1>/dev/null 2>/dev/null; then
# Usage: echo "${LOGS}" | log_lines LEVLE
log_lines() { local LEVEL="${1}"; while read -r LINE; do [ -z "${LINE}" ] && continue; log "${LEVEL}" "${LINE}"; done; }
fi
Expand Down
20 changes: 6 additions & 14 deletions src/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ load_libraries() {

_run_on_node() {
local HOST_NAME=
if ! HOST_NAME=$(docker node inspect self --format "{{.Description.Hostname}}" 2>&1); then
if ! HOST_NAME=$(run_cmd docker node inspect self --format "{{.Description.Hostname}}"); then
log DEBUG "Failed to run \"docker node inspect self\": ${HOST_NAME}"
return 1
fi
Expand All @@ -73,18 +73,9 @@ _run_on_node() {

_read_docker_hub_rate() {
local HOST PASSWORD USER
if ! PASSWORD=$(gantry_read_registry_password 2>&1); then
log ERROR "Failed to read registry PASSWORD: ${PASSWORD}";
PASSWORD=
fi
if ! USER=$(gantry_read_registry_username 2>&1); then
log ERROR "Failed to read registry USER: ${USER}";
USER=
fi
if ! HOST=$(gantry_read_registry_host 2>&1); then
log ERROR "Failed to read registry HOST: ${HOST}";
HOST=
fi
USER=$(gantry_read_config "GANTRY_REGISTRY_USER")
PASSWORD=$(gantry_read_config "GANTRY_REGISTRY_PASSWORD")
HOST=$(gantry_read_config "GANTRY_REGISTRY_HOST")
local USER_AND_PASS=
if [ -n "${USER}" ] && [ -n "${PASSWORD}" ]; then
if [ -z "${HOST}" ] || [ "${HOST}" = "docker.io" ]; then
Expand All @@ -107,6 +98,7 @@ gantry() {
START_TIME=$(date +%s)

[ -n "${DOCKER_HOST}" ] && log DEBUG "DOCKER_HOST=${DOCKER_HOST}"
[ -n "${DOCKER_CONFIG}" ] && log DEBUG "DOCKER_CONFIG=${DOCKER_CONFIG}"
local RUN_ON_NODE=
if ! RUN_ON_NODE=$(_run_on_node); then
local HOST_STRING="${DOCKER_HOST:-"the current node"}"
Expand All @@ -123,7 +115,7 @@ gantry() {
eval_cmd "pre-run" "${PRE_RUN_CMD}"
ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))

log INFO "Starting."
log INFO "Starting Gantry."
gantry_initialize "${STACK}"
ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))

Expand Down
87 changes: 54 additions & 33 deletions src/lib-common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ grep_q() {
# "grep -q" will exit immediately when the first line of data matches, and leading to broken pipe errors.
grep -q -- "${@}";
local GREP_RETURN=$?;
# Add "cat > /dev/null" to avoid broken pipe errors.
cat >/dev/null;
# Add "cat 1>/dev/null" to avoid broken pipe errors.
cat 1>/dev/null;
return "${GREP_RETURN}"
}

Expand All @@ -49,7 +49,7 @@ grep_q() {
grep_q_i() {
grep -q -i -- "${@}";
local GREP_RETURN=$?;
cat >/dev/null;
cat 1>/dev/null;
return "${GREP_RETURN}"
}

Expand Down Expand Up @@ -242,7 +242,7 @@ _log_docker_time() {
# date -d "${TIME_INPUT}" +"$(_time_format)" 2>/dev/null && return 0
local EPOCH=
if EPOCH=$(busybox date -d "${TIME_INPUT}" -D "%Y-%m-%dT%H:%M:%S" -u +%s 2>/dev/null); then
date -d "@${EPOCH}" +"$(_time_format)" 2>&1
date -d "@${EPOCH}" +"$(_time_format)"
return 0
fi
if [ -n "${TIME_INPUT}" ]; then
Expand Down Expand Up @@ -367,7 +367,7 @@ read_config() {
cat "${CONFIG_FILE}"
return $?
elif [ -n "${CONFIG_FILE}" ]; then
echo "Failed to read ${CONFIG_FILE}" >&2
echo "Failed to read file ${CONFIG_FILE}" >&2
return 1
fi
eval "local CONFIG=\${${CONFIG_NAME}}"
Expand Down Expand Up @@ -434,13 +434,36 @@ eval_cmd() {
return "${RETURN_VALUE}"
}

# When the command returns 0:
# Echo stdout and log stderr as a warning. Return 0.
# When the command returns non-zero:
# Echo stdout + stderr. Return the same value from the docker command.
run_cmd() {
local STDERR_STR=
local RETURN_VALUE=
# Use "3>&2 2>&1 1>&3" to swap stdout and stderr
{ STDERR_STR=$("${@}" 3>&2 2>&1 1>&3); } 2>&1
RETURN_VALUE=$?

if [ -n "${STDERR_STR}" ]; then
if [ "${RETURN_VALUE}" = 0 ]; then
log WARN "${STDERR_STR} (From command: ${*})"
else
echo "${STDERR_STR}"
fi
fi
return "${RETURN_VALUE}"
}

swarm_network_arguments() {
if [ -z "${NETWORK_NAME}" ]; then
echo ""
return 0
fi
NETWORK_NAME=$(docker network ls --filter "name=${NETWORK_NAME}" --format '{{.Name}}')
if [ -z "${NETWORK_NAME}" ]; then
local RETURN_VALUE=
NETWORK_NAME=$(run_cmd docker network ls --filter "name=${NETWORK_NAME}" --format '{{.Name}}')
RETURN_VALUE=$?
if [ "${RETURN_VALUE}" != "0" ] || [ -z "${NETWORK_NAME}" ]; then
echo ""
return 0
fi
Expand Down Expand Up @@ -481,7 +504,7 @@ docker_service_logs() {

_docker_service_exists() {
local SERVICE_NAME="${1}"
docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" >/dev/null 2>&1
docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" 1>/dev/null 2>/dev/null
}

_docker_wait_until_service_removed() {
Expand Down Expand Up @@ -513,8 +536,8 @@ _docker_service_task_states() {
local SERVICE_NAME="${1}"
# We won't get the return value of the command via $? if we use "local STATES=$(command)".
local STATES=
if ! STATES=$(docker service ps --no-trunc --format '[{{.Name}}][{{.Node}}] {{.CurrentState}} {{.Error}}' "${SERVICE_NAME}" 2>&1); then
echo "${STATES}" >&2
if ! STATES=$(run_cmd docker service ps --no-trunc --format '[{{.Name}}][{{.Node}}] {{.CurrentState}} {{.Error}}' "${SERVICE_NAME}"); then
log ERROR "${STATES}"
return 1
fi
local NAME_LIST=
Expand Down Expand Up @@ -591,7 +614,7 @@ wait_service_state() {
local RETURN_VALUE=0
local DOCKER_CMD_ERROR=1
local STATES=
while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do
while STATES=$(_docker_service_task_states "${SERVICE_NAME}"); do
DOCKER_CMD_ERROR=0
RETURN_VALUE=$(_all_tasks_reach_state "${WANT_STATE}" "${CHECK_FAILURES}" "${STATES}") && break
local SECONDS_ELAPSED=
Expand All @@ -606,7 +629,7 @@ wait_service_state() {
DOCKER_CMD_ERROR=1
done
if [ "${DOCKER_CMD_ERROR}" != "0" ]; then
log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${STATES}"
log ERROR "Failed to obtain task states of service ${SERVICE_NAME}."
return 1
fi
local LINE=
Expand All @@ -621,12 +644,10 @@ docker_service_remove() {
local POST_COMMAND="${2}"
! _docker_service_exists "${SERVICE_NAME}" && return 0
log DEBUG "Removing service ${SERVICE_NAME}."
local RETURN_VALUE=0
local LOG=
if ! LOG=$(docker service rm "${SERVICE_NAME}" 2>&1); then
RETURN_VALUE=$?
if ! LOG=$(run_cmd docker service rm "${SERVICE_NAME}"); then
log ERROR "Failed to remove docker service ${SERVICE_NAME}: ${LOG}"
return "${RETURN_VALUE}"
return 1
fi
if [ -n "${POST_COMMAND}" ]; then
eval "${POST_COMMAND}"
Expand Down Expand Up @@ -654,7 +675,7 @@ docker_global_job() {
SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
log INFO "Starting global-job ${SERVICE_NAME}."
local LOG=
if ! LOG=$(docker service create --mode global-job "${@}" 2>&1); then
if ! LOG=$(run_cmd docker service create --mode global-job "${@}"); then
log ERROR "Failed to create global-job ${SERVICE_NAME}: ${LOG}"
return 1
fi
Expand All @@ -669,7 +690,7 @@ docker_replicated_job() {
# The Docker CLI does not exit on failures.
log INFO "Starting replicated-job ${SERVICE_NAME}."
local LOG=
if ! LOG=$(docker service create --mode replicated-job --detach "${@}" 2>&1); then
if ! LOG=$(run_cmd docker service create --mode replicated-job --detach "${@}"); then
log ERROR "Failed to create replicated-job ${SERVICE_NAME}: ${LOG}"
return 1
fi
Expand All @@ -682,10 +703,10 @@ docker_replicated_job() {

docker_version() {
local cver capi sver sapi
if ! cver=$(docker version --format '{{.Client.Version}}' 2>&1); then log ERROR "${cver}"; cver="error"; fi
if ! capi=$(docker version --format '{{.Client.APIVersion}}' 2>&1); then log ERROR "${capi}"; capi="error"; fi
if ! sver=$(docker version --format '{{.Server.Version}}' 2>&1); then log ERROR "${sver}"; sver="error"; fi
if ! sapi=$(docker version --format '{{.Server.APIVersion}}' 2>&1); then log ERROR "${sapi}"; sapi="error"; fi
if ! cver=$(run_cmd docker version --format '{{.Client.Version}}'); then log ERROR "${cver}"; cver="error"; fi
if ! capi=$(run_cmd docker version --format '{{.Client.APIVersion}}'); then log ERROR "${capi}"; capi="error"; fi
if ! sver=$(run_cmd docker version --format '{{.Server.Version}}'); then log ERROR "${sver}"; sver="error"; fi
if ! sapi=$(run_cmd docker version --format '{{.Server.APIVersion}}'); then log ERROR "${sapi}"; sapi="error"; fi
echo "Docker version client ${cver} (API ${capi}) server ${sver} (API ${sapi})"
}

Expand All @@ -694,24 +715,24 @@ docker_version() {
# return 1 when there is an error.
docker_current_container_name() {
local ALL_NETWORKS=
ALL_NETWORKS=$(docker network ls --format '{{.ID}}') || return 1;
ALL_NETWORKS=$(run_cmd docker network ls --format '{{.ID}}') || return 1;
[ -z "${ALL_NETWORKS}" ] && return 0;
local IPS=;
# Get the string after "src":
# 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
IPS=$(ip route | grep src | sed -n -E "s/.* src (\S+).*$/\1/p");
[ -z "${IPS}" ] && return 0;
local GWBRIDGE_NETWORK HOST_NETWORK;
GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^docker_gwbridge$') || return 1;
HOST_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^host$') || return 1;
GWBRIDGE_NETWORK=$(run_cmd docker network ls --format '{{.ID}}' --filter 'name=^docker_gwbridge$') || return 1;
HOST_NETWORK=$(run_cmd docker network ls --format '{{.ID}}' --filter 'name=^host$') || return 1;
local NID=;
for NID in ${ALL_NETWORKS}; do
# The output of gwbridge does not contain the container name. It looks like gateway_8f55496ce4f1/172.18.0.5/16.
[ "${NID}" = "${GWBRIDGE_NETWORK}" ] && continue;
# The output of host does not contain an IP.
[ "${NID}" = "${HOST_NETWORK}" ] && continue;
local ALL_LOCAL_NAME_AND_IP=;
ALL_LOCAL_NAME_AND_IP=$(docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1;
ALL_LOCAL_NAME_AND_IP=$(run_cmd docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1;
local NAME_AND_IP=;
for NAME_AND_IP in ${ALL_LOCAL_NAME_AND_IP}; do
[ -z "${NAME_AND_IP}" ] && continue;
Expand Down Expand Up @@ -747,26 +768,26 @@ docker_remove() {
fi
log DEBUG "Removing container ${CNAME}."
if [ "${STATUS}" = "running" ]; then
docker stop "${CNAME}" >/dev/null 2>/dev/null
docker container stop "${CNAME}" 1>/dev/null 2>/dev/null
fi
# If the container is created with "--rm", it will be removed automatically when being stopped.
docker rm -f "${CNAME}" >/dev/null;
docker container rm -f "${CNAME}" 1>/dev/null;
log INFO "Removed container ${CNAME}."
}

docker_run() {
local RETRIES=0
local MAX_RETRIES=5
local SLEEP_SECONDS=10
local MSG=
while ! MSG=$(docker run "${@}" 2>&1); do
local LOG=
while ! LOG=$(run_cmd docker container run "${@}"); do
if [ ${RETRIES} -ge ${MAX_RETRIES} ]; then
log ERROR "Failed to run docker. Reached the max retries ${MAX_RETRIES}. ${MSG}"
log ERROR "Failed to run docker. Reached the max retries ${MAX_RETRIES}. ${LOG}"
return 1
fi
RETRIES=$((RETRIES + 1))
sleep ${SLEEP_SECONDS}
log WARN "Retry docker run (${RETRIES}). ${MSG}"
log WARN "Retry docker container run (${RETRIES}). ${LOG}"
done
echo "${MSG}"
echo "${LOG}"
}
Loading

0 comments on commit e003ede

Please sign in to comment.