[关闭]
@awsekfozc 2016-01-09T01:20:14.000000Z 字数 7544 阅读 2428

OpenTSDB 监控报警接入

OpenTSDB

参考：http://www.ttlsa.com/opentsdb/opentsdb-nagios-monitoring-and-alarming-realization/

1）脚本说明

在OpenTSDB源码tools目录下有一个Python工具check_tsd。该脚本查询OpenTSDB并返回兼容Nagios的输出OK/WARNING/CRITICAL状态格式。

  1. #!/usr/bin/python
  2. #
  3. # 该脚本根据给定的metric查询TSDB，并与指定的阀值进行比较。
  4. # 兼容Nagios的输出格式，因此可以作为nagios的命令。
  5. #
  6. # check_tsd -m mysql.slave.seconds_behind_master -t host=foo -t schema=mydb
  7. #     -d 600 -a avg -x gt -w 50 -c 100
  8. #
  10. import httplib
  11. import operator
  12. import socket
  13. import sys
  14. import time
  15. from optparse import OptionParser
  17. def main(argv):
  18.     """从TSDB提取数据，并做简单的Nagios报警。"""
  20.     parser = OptionParser(description='Simple TSDB data extractor for Nagios.')
  21.     parser.add_option('-H', '--host', default='localhost', metavar='HOST',
  22.             help='Hostname to use to connect to the TSD.')
  23.     parser.add_option('-p', '--port', type='int', default=80, metavar='PORT',
  24.             help='Port to connect to the TSD instance on.')
  25.     parser.add_option('-m', '--metric', metavar='METRIC',
  26.             help='Metric to query.')
  27.     parser.add_option('-t', '--tag', action='append', default=[],
  28.             metavar='TAG', help='Tags to filter the metric on.')
  29.     parser.add_option('-d', '--duration', type='int', default=600,
  30.             metavar='SECONDS', help='How far back to look for data.')
  31.     parser.add_option('-D', '--downsample', default='none', metavar='METHOD',
  32.             help='Downsample function, e.g. one of avg, min, sum, or max.')
  33.     parser.add_option('-W', '--downsample-window', type='int', default=60,
  34.             metavar='SECONDS', help='Window size over which to downsample.')
  35.     parser.add_option('-a', '--aggregator', default='sum', metavar='METHOD',
  36.             help='Aggregation method: avg, min, sum (default), max.')
  37.     parser.add_option('-x', '--method', dest='comparator', default='gt',
  38.             metavar='METHOD', help='Comparison method: gt, ge, lt, le, eq, ne.')
  39.     parser.add_option('-r', '--rate', default=False,
  40.             action='store_true', help='Use rate value as comparison operand.')
  41.     parser.add_option('-w', '--warning', type='float', metavar='THRESHOLD',
  42.             help='Threshold for warning.  Uses the comparison method.')
  43.     parser.add_option('-c', '--critical', type='float', metavar='THRESHOLD',
  44.             help='Threshold for critical.  Uses the comparison method.')
  45.     parser.add_option('-v', '--verbose', default=False,
  46.             action='store_true', help='Be more verbose.')
  47.     parser.add_option('-T', '--timeout', type='int', default=10,
  48.             metavar='SECONDS',
  49.             help='How long to wait for the response from TSD.')
  50.     parser.add_option('-E', '--no-result-ok', default=False,
  51.             action='store_true',
  52.             help='Return OK when TSD query returns no result.')
  53.     parser.add_option('-I', '--ignore-recent', default=0, type='int',
  54.             metavar='SECONDS', help='Ignore data points that are that'
  55.             ' are that recent.')
  56.     parser.add_option('-S', '--ssl', default=False, action='store_true',
  57.             help='Make queries to OpenTSDB via SSL (https)')
  58.     (options, args) = parser.parse_args(args=argv[1:])
  60.     # 验证参数
  61.     if options.comparator not in ('gt', 'ge', 'lt', 'le', 'eq', 'ne'):
  62.         parser.error("Comparator '%s' not valid." % options.comparator)
  63.     elif options.downsample not in ('none', 'avg', 'min', 'sum', 'max'):
  64.         parser.error("Downsample '%s' not valid." % options.downsample)
  65.     elif options.aggregator not in ('avg', 'min', 'sum', 'max'):
  66.         parser.error("Aggregator '%s' not valid." % options.aggregator)
  67.     elif not options.metric:
  68.         parser.error('You must specify a metric (option -m).')
  69.     elif options.duration <= 0:
  70.         parser.error('Duration must be strictly positive.')
  71.     elif options.downsample_window <= 0:
  72.         parser.error('Downsample window must be strictly positive.')
  73.     elif options.critical is None and options.warning is None:
  74.         parser.error('You must specify at least a warning threshold (-w) or a'
  75.                      ' critical threshold (-c).')
  76.     elif options.ignore_recent < 0:
  77.         parser.error('--ignore-recent must be positive.')
  79.     if not options.critical:
  80.         options.critical = options.warning
  81.     elif not options.warning:
  82.         options.warning = options.critical
  84.     # 处理标签
  85.     tags = ','.join(options.tag)
  86.     if tags:
  87.         tags = '{' + tags + '}'
  89.     # 组装URL并获取
  90.     if options.downsample == 'none':
  91.         downsampling = ''
  92.     else:
  93.         downsampling = '%ds-%s:' % (options.downsample_window,
  94.                                     options.downsample)
  95.     if options.rate:
  96.         rate = 'rate:'
  97.     else:
  98.         rate = ''
  99.     url = ('/q?start=%ss-ago&m=%s:%s%s%s%s&ascii&nagios'
  100.            % (options.duration, options.aggregator, downsampling, rate,
  101.               options.metric, tags))
  102.     tsd = '%s:%d' % (options.host, options.port)
  103.     if options.ssl:  # Pick the class to instantiate first.
  104.       conn = httplib.HTTPSConnection
  105.     else:
  106.       conn = httplib.HTTPConnection
  107.     if sys.version_info[0] * 10 + sys.version_info[1] >= 26:  # Python >2.6
  108.       conn = conn(tsd, timeout=options.timeout)
  109.     else:  # Python 2.5 or less, using the timeout kwarg will make it croak :(
  110.       conn = conn(tsd)
  111.     try:
  112.       conn.connect()
  113.     except socket.error, e:
  114.       print "ERROR: couldn't connect to %s: %s" % (tsd, e)
  115.       return 2
  116.     if options.verbose:
  117.         peer = conn.sock.getpeername()
  118.         print 'Connected to %s:%d' % (peer[0], peer[1])
  119.         conn.set_debuglevel(1)
  120.     now = int(time.time())
  121.     try:
  122.       conn.request('GET', url)
  123.       res = conn.getresponse()
  124.       datapoints = res.read()
  125.       conn.close()
  126.     except socket.error, e:
  127.       print "ERROR: couldn't GET %s from %s: %s" % (url, tsd, e)
  128.       return 2
  130.     # URL请求失败时
  131.     if res.status not in (200, 202):
  132.         print ('CRITICAL: status = %d when talking to %s:%d'
  133.                % (res.status, options.host, options.port))
  134.         if options.verbose:
  135.             print 'TSD said:'
  136.             print datapoints
  137.         return 2
  139.     # URL请求成功时
  140.     if options.verbose:
  141.         print datapoints
  142.     datapoints = datapoints.splitlines()
  144.     def no_data_point():
  145.  """从TSDB没有获取到任何数据时"""
  146.         if options.no_result_ok:
  147.             print 'OK: query did not return any data point (--no-result-ok)'
  148.             return 0
  149.         else:
  150.             print 'CRITICAL: query did not return any data point'
  151.             return 2
  153.     if not len(datapoints):
  154.         return no_data_point()
  156.     comparator = operator.__dict__[options.comparator]
  157.     rv = 0         # 该脚本返回值，0-OK,1-WARNING,2-CRITICAL
  158.     badts = None   # 超过阀值的时间戳
  159.     badval = None  # 超过阀值的值
  160.     npoints = 0    # 查询到多少数据点？
  161.     nbad = 0       # 有多少数据点超过阀值？
  162.     for datapoint in datapoints:
  163.         datapoint = datapoint.split()
  164.         ts = int(datapoint[1])
  165.         delta = now - ts
  166.         if delta > options.duration or delta <= options.ignore_recent:
  167.             continue  # 忽略不在options.duration或options.ignore_recent的数据点
  168.         npoints += 1
  169.         val = datapoint[2]
  170.         if '.' in val:
  171.             val = float(val)
  172.         else:
  173.             val = int(val)
  174.         bad = False  # Is the current value bad?
  175.         # 比较 warning/crit
  176.         if comparator(val, options.critical):
  177.             rv = 2
  178.             bad = True
  179.             nbad += 1
  180.         elif rv < 2 and comparator(val, options.warning):
  181.             rv = 1
  182.             bad = True
  183.             nbad += 1
  184.         if (bad and
  185.             (badval is None  # First bad value we find.
  186.              or comparator(val, badval))):  # Worse value.
  187.             badval = val
  188.             badts = ts
  190.     if options.verbose and len(datapoints) != npoints:
  191.         print ('ignored %d/%d data points for being more than %ds old'
  192.                % (len(datapoints) - npoints, len(datapoints), options.duration))  #忽略量/总查询量 数据点大于duration时间
  193.     if not npoints:
  194.         return no_data_point()
  195.     if badts:
  196.         if options.verbose:
  197.             print 'worse data point value=%s at ts=%s' % (badval, badts)  # 糟糕数据点的值以及该点值的时间戳
  198.         badts = time.asctime(time.localtime(badts))
  200.     # 在NRPE里，字符串'|'有特殊含义，但在tag搜索时有用到。对其替换。
  201.     ttags = tags.replace("|",":")
  202.     if not rv:
  203.         print ('OK: %s%s: %d values OK, last=%r'
  204.                % (options.metric, ttags, npoints, val))  # OK: metric{tags}: 数据点数量 values OK, 最后一次的值
  205.     else:
  206.         if rv == 1:
  207.             level = 'WARNING'
  208.             threshold = options.warning
  209.         elif rv == 2:
  210.             level = 'CRITICAL'
  211.             threshold = options.critical
  212.         print ('%s: %s%s %s %s: %d/%d bad values (%.1f%%) worst: %r @ %s'  
  213.                % (level, options.metric, ttags, options.comparator, threshold,
  214.                   nbad, npoints, nbad * 100.0 / npoints, badval, badts))
  215.     return rv
  218. if __name__ == '__main__':
  219.     sys.exit(main(sys.argv))

2）参数说明

  1. Options:
  2.   -h, --help            打印帮助信息
  3.   -H HOST, --host=HOST  指定TSDB服务器
  4.   -p PORT, --port=PORT  指定TSDB端口
  5.   -m METRIC, --metric=METRIC
  6.                         查询的metric
  7.   -t TAG, --tag=TAG     metric过滤的tags
  8.   -d SECONDS, --duration=SECONDS
  9.                         查询多久的数据，默认600s
  10.   -D METHOD, --downsample=METHOD
  11.                         缩减像素采样函数有avg, min, sum, or max.
  12.   -W SECONDS, --downsample-window=SECONDS
  13.                         缩减像素采样窗口大小
  14.   -a METHOD, --aggregator=METHOD
  15.                         聚合方法: avg, min, sum (default), max.
  16.   -x METHOD, --method=METHOD
  17.                         比较法: gt, ge, lt, le, eq, ne.
  18.   -r, --rate            使用速率值作为比较的操作数
  19.   -w THRESHOLD, --warning=THRESHOLD
  20.                         warning阀值.
  21.   -c THRESHOLD, --critical=THRESHOLD
  22.                         critical阀值.
  23.   -v, --verbose         显示详细信息
  24.   -T SECONDS, --timeout=SECONDS
  25.                         等待TSDB响应时间
  26.   -E, --no-result-ok    当TSD查询没有结果返回时，返回OK状态
  27.   -I SECONDS, --ignore-recent=SECONDS
  28.                         忽略最近的数据点
  29.   -S, --ssl             通过SSL查询OpenTSDB (https)

3）报警接入

Nagios服务器搭建

将check_tsd脚本放在Nagios服务器/usr/local/nagios/libexec目录里。
编辑/usr/local/nagios/etc/objects/commands.cnf文件，添加下面的命令：

  1. define command{
  2.  command_name check_tsd
  3.  command_line $USER1$/check_tsd -H 10.0.101.145 -p 4242 -t host=$HOSTADDRESS$ $ARG1$
  4. }
  6. ###添加监控项
  7. define service {
  8.     use                         MongoDB
  9.     host_name                   10.0.0.166
  10.     service_description         query opcounters per second
  11.     check_command               check_tsd!-m rate:mongo.opcounters -t type=query -d 60 -w 300 -c 500
  12. }

在此输入正文

添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注