-
František Řezníček authoredFrantišek Řezníček authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
metric-generator-exec.sh 5.47 KiB
#!/usr/bin/env bash
# metric-generator-exec.sh
#
# Usage:
# metric-generator-exec.sh <metric-generator-module-name> <metrics-generator-output-textfile-metric-file> <runner-output-textfile-metric-file>
#
# Executes specified metric generator module <metric-generator-module-name>
# writes following text metric files:
# * <metrics-generator-output-textfile-metric-file> generated by the <metric-generator-module-name>
# * <runner-output-textfile-metric-file> generated by this runner (measuring <metric-generator-module-name> is successfully executed)
#
# Requirements:
# 1. CMG module has to exit with ecode of 0
# 2. CMG module has to produce metrics on stdout (empty stdout is also invalid, generate at least metric help comments)
#
# Examples:
# # launch `smartmon` metric generator module
# $ metric-generator-exec.sh smartmon /tmp/smartmon.prom /tmp/cmg-smartmon.prom
set -eo pipefail
# arguments
METRICS_GENERATOR_NAME="$1"
OUTPUT_METRIC_GENERATOR_METRIC_FILE="$2"
OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE="$3"
# constants
CMG_SRC_DIR=$(dirname $(readlink -f $0))
CMG_METRIC_GENERATOR_EXEC_TIMEOUT="40"
METRICS_GENERATOR_FILE="$(ls ${CMG_SRC_DIR}/metric-generators/${METRICS_GENERATOR_NAME}* | head -1)"
GENERATOR_START_TIME=$(date +%s)
STAGE_NAME="configuration loaded"
source "${CMG_SRC_DIR}/../custom-metrics-generator.conf.env"
STAGE_NAME="library loaded"
source "${CMG_SRC_DIR}/lib.sh"
# get_metric_text( <metric-name> <metric-generator-name> <metric-timestap-value> )
# generates metric text
function get_metric_text() {
local metric_name="$1"
local metric_generator_name="$2"
local metric_timestamp_value="$3"
if [ "${metric_name}" == "job_last_run_timestamp" ]; then
get_metric_help "${metric_name}" "gauge" "Last job execution timestamp."
printf '%s{app="%s",generator="%s"} %d\n' "${metric_name}" "custom-metrics-generator" "${metric_generator_name}" "${metric_timestamp_value}"
elif [ "${metric_name}" == "job_last_successful_run_timestamp" ]; then
get_metric_help "${metric_name}" "gauge" "Last successful job execution timestamp."
printf '%s{app="%s",generator="%s"} %d\n' "${metric_name}" "custom-metrics-generator" "${metric_generator_name}" "${metric_timestamp_value}"
else
log_stderr "ERROR: get_metric_text() does not know how to generate ${metric_name} metric"
false
fi
}
# at_exit()
# callback procedure executed at exit (trap)
function at_exit() {
set +e
# handling partial <runner-output-textfile-metric-file> data
if [ -e "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp" ]; then
if ! grep -q "^job_last_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"; then
if grep -q "^job_last_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"; then
grep "job_last_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
else
get_metric_text "job_last_run_timestamp" "${METRICS_GENERATOR_NAME}" 0 >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
fi
else
if ! grep -q "^job_last_successful_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"; then
if grep -q "^job_last_successful_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"; then
grep "job_last_successful_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
else
get_metric_text "job_last_successful_run_timestamp" "${METRICS_GENERATOR_NAME}" 0 >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
fi
fi
fi
mv -f "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"
fi
set > "/tmp/cmg-${METRICS_GENERATOR_NAME}-trace.log"
if [ "${STAGE_NAME}" != "success" ]; then
log_stderr "ERROR: Job ${METRICS_GENERATOR_NAME} failed at step \"${STAGE_NAME}\" (PPID: $$, duration: ${SECONDS} sec[s], see env. dump at /tmp/cmg-*${METRICS_GENERATOR_NAME}-trace.log)."
exit 1
fi
log_stdout "INFO: Job ${METRICS_GENERATOR_NAME} succeeded. (PPID: $$, duration: ${SECONDS} sec[s])"
}
trap at_exit EXIT
log_stdout "INFO: Job ${METRICS_GENERATOR_NAME} started. (PPID: $$)"
STAGE_NAME="job_last_run_timestamp metric generated"
rm -f "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
get_metric_text "job_last_run_timestamp" "${METRICS_GENERATOR_NAME}" "${GENERATOR_START_TIME}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
STAGE_NAME="metric-generator successfuly executed"
timeout ${CMG_METRIC_GENERATOR_EXEC_TIMEOUT} ${METRICS_GENERATOR_FILE} ${OUTPUT_METRIC_GENERATOR_METRIC_FILE} \
> "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}.tmp" \
2> "/tmp/cmg-module-${METRICS_GENERATOR_NAME}-trace.log"
STAGE_NAME="metric-generator generated valid metrics"
test -s "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}.tmp"
STAGE_NAME="metric-generator atomicly wrote generated metrics"
mv -f "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}.tmp" "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}"
STAGE_NAME="job_last_run_timestamp metric generated"
GENERATOR_END_TIME=$(date +%s)
get_metric_text "job_last_successful_run_timestamp" "${METRICS_GENERATOR_NAME}" "${GENERATOR_END_TIME}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
mv -f "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"
STAGE_NAME="success"