---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-script-config
namespace: ops-department
labels:
k8s-app: filebeat
data:
set-kafka-topic.js: |
function process(event) {
if (event.Get("kubernetes.namespace")) {
event.Put("kafka_topic", "log-k8s-" + event.Get("kubernetes.namespace"));
} else {
throw new Error("Kubernetes namespace is not defined.");
}
return event;
}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: ops-department
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
http.enabled: true
http.host: 0.0.0.0
http.port: 5066
filebeat.inputs:
- type: container
stream: stdout
paths:
- /var/log/containers/*.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
in_cluster: true
default_matchers.enabled: true
matchers:
- logs_path:
logs_path: "/var/log/containers/"
- rename:
fields:
- from: message
to: "@message"
- from: source
to: "@path"
- from: node_name
to: "@hostname"
- from: "kubernetes.namespace_name"
to: "kubernetes_namespace"
ignore_missing: true
- script:
lang: javascript
id: "set_kafka_topic"
file: "/usr/share/filebeat/scripts/set-kafka-topic.js"
ignore_imssing: true
processors:
- add_cloud_metadata:
exclude_fields: ["host"]
- add_host_metadata:
exclude_fields: ["host"]
- decode_json_fields:
fields: ["message"]
target: ""
overwrite_keys: true
add_error_key: true
- if:
contains:
message: kafka_topic
then:
- dissect:
tokenizer: "%{[@metadata][beat]} %{[@metadata][version]} [%{loglevel}] [%{module}] [%{namespace}] [%{podname}] %{[@metadata][message]} kafka_topic:%{kafka_topic}"
field: "message"
target_prefix: ""
- rename:
fields:
- { from: "kafka_topic", to: "topic" }
- drop_fields:
fields: ["beat", "input", "prospector.type", "offset", "source", "log", "ecs", "host", "container", "agent", "cloud", "tags", "kubernetes.replicaset", "kubernetes.labels", "kubernetes.namespace labels", "kubernetes.container", "kubernetes.node", "kubernetes.namespace_labels"]
ignore_missing: true
- drop_event:
when:
not:
or:
- equals:
kubernetes.namespace: "ops-department"
- equals:
kubernetes.namespace: "account"
- equals:
kubernetes.namespace: "jinjiang-online"
output.kafka:
enabled: true
hosts:
- 10.10.10.10:9092
topic: "%{[kafka_topic]}"
protocol_version: "2.0.0"
compression: gzip
max_message_bytes: 1000000
multiline:
pattern: ^\d{4}-\d{2}-\d{2}
negate: true
match: after
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: ops-department
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.17.9
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
resources:
limits:
cpu: 200m
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: scripts
mountPath: /usr/share/filebeat/scripts
readOnly: true
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
- name: host-time
mountPath: /etc/localtime
volumes:
- name: scripts
configMap:
name: filebeat-script-config
- name: config
configMap:
defaultMode: 0640
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
- name: data
hostPath:
# When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
path: /var/lib/filebeat-data
type: DirectoryOrCreate
- name: host-time
hostPath:
path: /etc/localtime
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: ops-department
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: filebeat
namespace: ops-department
subjects:
- kind: ServiceAccount
name: filebeat
namespace: ops-department
roleRef:
kind: Role
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: filebeat-kubeadm-config
namespace: ops-department
subjects:
- kind: ServiceAccount
name: filebeat
namespace: ops-department
roleRef:
kind: Role
name: filebeat-kubeadm-config
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
- nodes
verbs:
- get
- watch
- list
- apiGroups: ["apps"]
resources:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: filebeat
# should be the namespace where filebeat is running
namespace: ops-department
labels:
k8s-app: filebeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: filebeat-kubeadm-config
namespace: ops-department
labels:
k8s-app: filebeat
rules:
- apiGroups: [""]
resources:
- configmaps
resourceNames:
- kubeadm-config
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: ops-department
labels:
k8s-app: filebeat
---