Hadoop-HDFS 监控指标采集

简介

采集 hadoop-hdfs 指标上报到 dataflux 中

前置条件

• 已安装 DataKit（DataKit 安装文档）
• 已配置 Jolokia 指标采集

配置

进入 DataKit 安装目录下的 conf.d/hadoop_hdfs 目录，复制 hadoop_hdfs.conf.sample 并命名为 hadoop_hdfs.conf。示例如下：

[[inputs.jolokia2_agent]]
  urls = ["http://localhost:8778/jolokia"]
  name_prefix = "hadoop.hdfs.namenode."

  [[inputs.jolokia2_agent.metric]]
    name = "FSNamesystem"
    mbean = "Hadoop:name=FSNamesystem,service=NameNode"
    paths = ["CapacityTotal", "CapacityRemaining", "CapacityUsedNonDFS", "NumLiveDataNodes", "NumDeadDataNodes", "NumInMaintenanceDeadDataNodes", "NumDecomDeadDataNodes"]

  [[inputs.jolokia2_agent.metric]]
    name = "FSNamesystemState"
    mbean = "Hadoop:name=FSNamesystemState,service=NameNode"
    paths = ["VolumeFailuresTotal", "UnderReplicatedBlocks", "BlocksTotal"]
  
  [[inputs.jolokia2_agent.metric]]
    name = "OperatingSystem"
    mbean = "java.lang:type=OperatingSystem"
    paths = ["ProcessCpuLoad", "SystemLoadAverage", "SystemCpuLoad"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_runtime"
    mbean = "java.lang:type=Runtime"
    paths = ["Uptime"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_memory"
    mbean = "java.lang:type=Memory"
    paths = ["HeapMemoryUsage", "NonHeapMemoryUsage", "ObjectPendingFinalizationCount"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_garbage_collector"
    mbean = "java.lang:name=*,type=GarbageCollector"
    paths = ["CollectionTime", "CollectionCount"]
    tag_keys = ["name"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_memory_pool"
    mbean = "java.lang:name=*,type=MemoryPool"
    paths = ["Usage", "PeakUsage", "CollectionUsage"]
    tag_keys = ["name"]
    tag_prefix = "pool_"


################
# DATANODE     #
################
[[inputs.jolokia2_agent]]
  urls = ["http://localhost:7778/jolokia"]
  name_prefix = "hadoop.hdfs.datanode."

  [[inputs.jolokia2_agent.metric]]
    name = "FSDatasetState"
    mbean = "Hadoop:name=FSDatasetState,service=DataNode"
    paths = ["Capacity", "DfsUsed", "Remaining", "NumBlocksFailedToUnCache", "NumBlocksFailedToCache", "NumBlocksCached"]

  [[inputs.jolokia2_agent.metric]]
    name = "OperatingSystem"
    mbean = "java.lang:type=OperatingSystem"
    paths = ["ProcessCpuLoad", "SystemLoadAverage", "SystemCpuLoad"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_runtime"
    mbean = "java.lang:type=Runtime"
    paths = ["Uptime"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_memory"
    mbean = "java.lang:type=Memory"
    paths = ["HeapMemoryUsage", "NonHeapMemoryUsage", "ObjectPendingFinalizationCount"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_garbage_collector"
    mbean = "java.lang:name=*,type=GarbageCollector"
    paths = ["CollectionTime", "CollectionCount"]
    tag_keys = ["name"]

  [[inputs.jolokia2_agent.metric]]
    name = "jvm_memory_pool"
    mbean = "java.lang:name=*,type=MemoryPool"
    paths = ["Usage", "PeakUsage", "CollectionUsage"]
    tag_keys = ["name"]
    tag_prefix = "pool_"
                                 

配置好后，重启 DataKit 即可生效