Skip to content
Snippets Groups Projects
metric-generator-exec.sh 5.47 KiB
#!/usr/bin/env bash
# metric-generator-exec.sh
#
# Usage:
#   metric-generator-exec.sh <metric-generator-module-name> <metrics-generator-output-textfile-metric-file> <runner-output-textfile-metric-file>
#
# Executes specified metric generator module <metric-generator-module-name>
#   writes following text metric files:
#   * <metrics-generator-output-textfile-metric-file> generated by the <metric-generator-module-name>
#   * <runner-output-textfile-metric-file> generated by this runner (measuring <metric-generator-module-name> is successfully executed)
#
# Requirements:
#  1. CMG module has to exit with ecode of 0
#  2. CMG module has to produce metrics on stdout (empty stdout is also invalid, generate at least metric help comments)
#
# Examples:
#  # launch `smartmon` metric generator module
#  $ metric-generator-exec.sh smartmon /tmp/smartmon.prom /tmp/cmg-smartmon.prom

set -eo pipefail

# arguments
METRICS_GENERATOR_NAME="$1"
OUTPUT_METRIC_GENERATOR_METRIC_FILE="$2"
OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE="$3"

# constants
CMG_SRC_DIR=$(dirname $(readlink -f $0))
CMG_METRIC_GENERATOR_EXEC_TIMEOUT="40"
METRICS_GENERATOR_FILE="$(ls ${CMG_SRC_DIR}/metric-generators/${METRICS_GENERATOR_NAME}* | head -1)"
GENERATOR_START_TIME=$(date +%s)

STAGE_NAME="configuration loaded"
source "${CMG_SRC_DIR}/../custom-metrics-generator.conf.env"

STAGE_NAME="library loaded"
source "${CMG_SRC_DIR}/lib.sh"

# get_metric_text( <metric-name> <metric-generator-name> <metric-timestap-value> )
#   generates metric text
function get_metric_text() {
    local metric_name="$1"
    local metric_generator_name="$2"
    local metric_timestamp_value="$3"

    if [ "${metric_name}" == "job_last_run_timestamp" ]; then
        get_metric_help "${metric_name}" "gauge" "Last job execution timestamp."
        printf '%s{app="%s",generator="%s"} %d\n' "${metric_name}" "custom-metrics-generator" "${metric_generator_name}" "${metric_timestamp_value}"
    elif [ "${metric_name}" == "job_last_successful_run_timestamp" ]; then
        get_metric_help "${metric_name}" "gauge" "Last successful job execution timestamp."
        printf '%s{app="%s",generator="%s"} %d\n' "${metric_name}" "custom-metrics-generator" "${metric_generator_name}" "${metric_timestamp_value}"
    else
        log_stderr "ERROR: get_metric_text() does not know how to generate ${metric_name} metric"
        false
    fi
}

# at_exit()
#   callback procedure executed at exit (trap)
function at_exit() {
    set +e
    # handling partial <runner-output-textfile-metric-file> data
    if [ -e "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp" ]; then
        if ! grep -q "^job_last_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"; then
            if grep -q "^job_last_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"; then
                grep "job_last_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
            else
                get_metric_text "job_last_run_timestamp" "${METRICS_GENERATOR_NAME}" 0 >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
            fi
        else
            if ! grep -q "^job_last_successful_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"; then
                if grep -q "^job_last_successful_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"; then
                    grep "job_last_successful_run_timestamp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
                else
                    get_metric_text "job_last_successful_run_timestamp" "${METRICS_GENERATOR_NAME}" 0 >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
                fi
            fi
        fi
        mv -f "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"
    fi

    set > "/tmp/cmg-${METRICS_GENERATOR_NAME}-trace.log"
    if [ "${STAGE_NAME}" != "success" ]; then
        log_stderr "ERROR: Job ${METRICS_GENERATOR_NAME} failed at step \"${STAGE_NAME}\" (PPID: $$, duration: ${SECONDS} sec[s], see env. dump at /tmp/cmg-*${METRICS_GENERATOR_NAME}-trace.log)."
        exit 1
    fi
    log_stdout "INFO: Job ${METRICS_GENERATOR_NAME} succeeded. (PPID: $$, duration: ${SECONDS} sec[s])"
}

trap at_exit EXIT

log_stdout "INFO: Job ${METRICS_GENERATOR_NAME} started. (PPID: $$)"

STAGE_NAME="job_last_run_timestamp metric generated"
rm -f "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
get_metric_text "job_last_run_timestamp" "${METRICS_GENERATOR_NAME}" "${GENERATOR_START_TIME}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"

STAGE_NAME="metric-generator successfuly executed"
timeout ${CMG_METRIC_GENERATOR_EXEC_TIMEOUT} ${METRICS_GENERATOR_FILE} ${OUTPUT_METRIC_GENERATOR_METRIC_FILE} \
   > "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}.tmp" \
  2> "/tmp/cmg-module-${METRICS_GENERATOR_NAME}-trace.log"

STAGE_NAME="metric-generator generated valid metrics"
test -s "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}.tmp"

STAGE_NAME="metric-generator atomicly wrote generated metrics"
mv -f "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}.tmp" "${OUTPUT_METRIC_GENERATOR_METRIC_FILE}"

STAGE_NAME="job_last_run_timestamp metric generated"
GENERATOR_END_TIME=$(date +%s)
get_metric_text "job_last_successful_run_timestamp" "${METRICS_GENERATOR_NAME}" "${GENERATOR_END_TIME}" >> "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp"
mv -f "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}.tmp" "${OUTPUT_METRIC_GENERATOR_RUNNER_METRIC_FILE}"

STAGE_NAME="success"