JZTXT

Ceph存储日志收集、过滤和分析

发布时间 2023-08-23 12:19:52作者: Varden

一、方案简述

存储服务组件众多，且容器化多服务实例部署后，日志分散，需要聚合分析，使用 filebeat 来收集节点系统日志、Ceph守护进程实例日志和容器日志，推送至 ELK 集群集中过滤、转换和分析，提高故障排查效率。

二、方案架构图

三、测试环境部署

1、部署单节点ES

容器化部署脚本：

# cat deploy_es.sh
#!/bin/bash
#
docker run \
  --name es01-test \
  -d --restart=always \
  --net elastic \
  -p 9200:9200 \
  -p 9300:9300 \
  -e "discovery.type=single-node" \
  docker.elastic.co/elasticsearch/elasticsearch:7.17.12

2、部署Kibana

容器化部署脚本：

# cat deploy_kibana.sh
#!/bin/bash
#
docker run \
  --name kib01-test \
  -d --restart=always \
  --net elastic \
  -p 5601:5601 \
  -e "ELASTICSEARCH_HOSTS=http://es01-test:9200" \
  docker.elastic.co/kibana/kibana:7.17.12

3、部署Logstash

安装包：

logstash-7.17.12-x86_64.rpm

二进制部署：

rpm -ivh logstash-7.17.12-x86_64.rpm

 

配置示例：

# cat /etc/logstash/conf.d/es-pipeline.conf
input {
  beats {
    port => 5044
  }
}
 
output {
  elasticsearch {
    hosts => ["http://172.16.0.1:9200"]
    index => "ceph-%{[fields][dc]}-%{[fields][env]}-%{+YYYY.MM.dd}"
    # user => "elastic"
    # action => "create"
    # ilm_enabled => true
    # password => "xxxxx"
  }
}
 
# cat /etc/logstash/conf.d/filter.conf
filter {
  if "syslog" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{SYSLOGBASE} %{GREEDYDATA:log_message}"
        ]
      }
    }
  }
 
  else if "ceph-log" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:logsource} %{NOTSPACE:client} %{NOTSPACE:client_ip} %{NOTSPACE} : %{NOTSPACE:program} \[%{NOTSPACE:log_level}\] %{GREEDYDATA:log_message}"
        ]
      }
    }
  }
 
  else if "ceph-audit" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:logsource} %{NOTSPACE:client} %{NOTSPACE:client_ip} %{NOTSPACE} : %{NOTSPACE:program} \[%{NOTSPACE:log_level}\] %{GREEDYDATA:log_message}"
        ]
      }
    }
  }
 
  else if "ceph-mgr" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}"
        ]
      }
    }
  }
 
  else if "ceph-rgw" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}"
        ]
      }
    }
  }
 
  else if "ceph-mds" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}"
        ]
      }
    }
  }
 
  else if "ceph-mon" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}"
        ]
      }
    }
  }
 
  else if "ceph-osd" in [tags] {
    grok {
      match => {
        "message" => [
          "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}"
        ]
      }
    }
  }
}

启动服务：

systemctl start logstash
systemctl enable logstash

4、部署Filebeat

安装包：

filebeat-7.17.12-x86_64.rpm

二进制部署：

rpm -ivh filebeat-7.17.12-x86_64.rpm

配置示例：

# cat /etc/filebeat/filebeat.yml
filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false
 
setup.template:
  settings:
    index.number_of_shards: 1
  name: "ceph"
  pattern: "ceph-*"
  enabled: true
  overwrite: true
 
setup.ilm:
  enabled: false
 
#filebeat.autodiscover:
#  providers:
#    - type: docker
#      hints.enabled: true
 
#processors:
#- add_cloud_metadata: ~
#- add_docker_metadata: ~
#- add_host_metadata: ~
 
#output.elasticsearch:
#  hosts: 172.16.0.1:9200
#  username: ''
#  password: ''
#  index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}"
#
output.logstash:
  hosts: ["172.16.0.1:5044"]
#  index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}"
 
setup.kibana:
  host: 172.16.0.1:5601
 
fields:
  env: prod
  dc: guangming
 
tags: ["ceph","guangming","prod"]
 
filebeat.inputs:
# syslog
#- type: filestream
#  id: syslog-filestream-id
#  enabled: true
#  paths:
#    - /var/log/syslog
#    - /var/log/messages
#  fields:
#    log_source: syslog
#  tags: ["syslog"]
#  exclude_lines: ['.*systemd\[\d+\].*','.*systemd-resolved\[\d+\].*','.*ansible-.*','.*filebeat\[\d+\].*']
 
# ceph
- type: filestream
  id: ceph-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph.log
  fields:
    log_source: ceph-log
  tags: ["ceph-log"]
 
- type: filestream
  id: ceph-audit-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph.audit.log
  fields:
    log_source: ceph-audit
  tags: ["ceph-audit"]
 
- type: filestream
  id: ceph-mds-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph-mds.*.log
  fields:
    log_source: ceph-mds
  tags: ["ceph-mds"]
 
- type: filestream
  id: ceph-osd-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph-osd.*.log
  fields:
    log_source: ceph-osd
  tags: ["ceph-osd"]
 
- type: filestream
  id: ceph-mon-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph-mon.*.log
  fields:
    log_source: ceph-mon
  tags: ["ceph-mon"]
 
- type: filestream
  id: ceph-mgr-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph-mgr.*.log
  fields:
    log_source: ceph-mgr
  tags: ["ceph-mgr"]
 
- type: filestream
  id: ceph-rgw-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph-client.rgw.*.log
  fields:
    log_source: ceph-rgw
  tags: ["ceph-rgw"]
 
- type: filestream
  id: ceph-volume-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph-volume.log
  fields:
    log_source: ceph-volume
  tags: ["ceph-volume"]
 
- type: filestream
  id: ceph-volume-systemd-filestream-id
  enabled: true
  paths:
    - /var/log/ceph/ceph-volume-systemd.log
  fields:
    log_source: ceph-volume-systemd
  tags: ["ceph-volume-systemd"]

启动服务：

systemctl start filebeat
systemctl enable filebeat

 

容器化部署：

## 配置示例
# cat filebeat.docker.yml
filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false
 
setup.template:
  settings:
    index.number_of_shards: 1
  name: "ceph"
  pattern: "ceph-*"
  enabled: true
  overwrite: true
 
setup.ilm:
  enabled: false
 
filebeat.autodiscover:
  providers:
    - type: docker
      hints.enabled: true
 
processors:
#- add_cloud_metadata: ~
#- add_docker_metadata: ~
#- add_host_metadata: ~
 
#output.elasticsearch:
#  hosts: '172.16.0.1:9200'
#  username: ''
#  password: ''
#  index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}"
 
output.logstash:
  hosts: ["172.16.0.1:5044"]
#  index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}"
 
setup.kibana:
  host: "172.16.0.1:5601"
 
filebeat.inputs:
#- type: filestream
#  id: ceph-filestream-id
#  enabled: true
#  paths:
#    - /opt/log/messages
#  fields:
#    log_source: syslog
#  tags: ["syslog"]
#  exclude_lines: ['.*systemd\[\d+\].*','.*systemd-resolved\[\d+\].*','.*ansible-.*','.*filebeat\[\d+\].*']
 
fields:
  env: pre
  dc: guangming
 
tags: ["ceph","guangming","pre","docker"]
 
## 部署脚本
# cat deploy_filebeat.sh
#!/bin/bash
#
docker run -d \
--name=filebeat \
--restart=always \
--net=host \
--user=root \
--volume="$(pwd)/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro" \
--volume="/var/lib/docker/containers:/var/lib/docker/containers:ro" \
--volume="/var/run/docker.sock:/var/run/docker.sock:ro" \
--volume="/var/log:/opt/log:ro" \
docker.elastic.co/beats/filebeat:7.17.12 filebeat -e --strict.perms=false