Filebeat es 同步服务器日志到es的方法

1

2

3

4

5

6

7

8

9

10

11

12

内网地址

192.168.0.94:9200

127.0.0.1:9200

https://127.0.0.1:9200

账户 admin

密码 123456

#端口

9200 es

kibana

https://127.0.0.1:5601/app/login?nextUrl=%2F

账户 admin

密码 123456

1

5601,9200

1

2

3

4

5

6

# 1. 设置内核映射限制参数

sudo sysctl -w vm.max_map_count=262144

# 2. 永久写入配置

echo "vm.max_map_count=262144" | sudo tee -a /etc/sysctl.conf

# 3. 使配置生效

sudo sysctl -p

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

# 创建基础目录

sudo mkdir -p /www/es-kibana/{metricbeat/modules.d,metricbeat/config,elasticsearch/config,elasticsearch/data,elasticsearch/logs,kibana/config,kibana/logs}

# 拷贝或新建配置文件

# （如果之前已经编辑过，直接 mv 到相应目录即可）

# Elasticsearch 配置

sudo tee /www/es-kibana/elasticsearch/config/elasticsearch.yml > /dev/null << EOF

cluster.name: "es-docker-cluster"

network.host: 0.0.0.0

http.port: 9200

discovery.type: single-node

bootstrap.memory_lock: true

path.data: /usr/share/elasticsearch/data

path.logs: /usr/share/elasticsearch/logs

# ─── 安全认证 ───────────────────────────

xpack.security.enabled: true

# ─── 开启匿名访问（允许无凭据访问 ES HTTP 接口） ───────────────────────────

xpack.security.authc.anonymous.username: anonymous_user

xpack.security.authc.anonymous.roles: superuser

xpack.security.authc.anonymous.authz_exception: false

EOF

# Kibana 配置

sudo tee /www/es-kibana/kibana/config/kibana.yml > /dev/null << EOF

server.name: kibana

server.host: "0.0.0.0"

server.port: 5601

elasticsearch.hosts: [ "http://elasticsearch:9200" ]

elasticsearch.username: "elastic"

elasticsearch.password: "123456"

# 会话加密与安全相关

xpack.security.encryptionKey: "a_very_long_random_string_at_least_32_chars"

xpack.security.session.idleTimeout: "1h"

i18n.locale: "zh-CN"

logging.dest: /usr/share/kibana/logs/kibana.log

EOF

#Metricbeat 配置

sudo tee /www/es-kibana/metricbeat/config/metricbeat.yml > /dev/null << EOF

metricbeat.config.modules:

  path: /usr/share/metricbeat/modules.d/*.yml

  reload.enabled: false

setup.ilm.enabled: false

setup.template.enabled: true

setup.template.name: "metricbeat-mian-stg"

setup.template.pattern: "metricbeat-mian-stg-*"

output.elasticsearch:

  hosts: ["http://elasticsearch:9200"]

  username: "elastic"

  password: "123456"

monitoring.enabled: true

EOF

#启用默认系统监控模块

sudo tee /www/es-kibana/metricbeat/modules.d/system.yml > /dev/null << EOF

- module: system

  metricsets:

    - cpu

    - load

    - memory

    - network

    - process

    - process_summary

    - uptime

    - filesystem

    - diskio

    - socket_summary

  period: 10s

  processes: ['.*']

  enabled: true

EOF

# 确保目录权限（Elasticsearch 默认 UID/GID 都是 1000）

sudo chown -R 1000:1000 /www/es-kibana/elasticsearch/{data,logs}

sudo chown -R 1000:1000 /www/es-kibana/kibana/logs

cd /www/es-kibana

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

version: '3.8'

services:

  elasticsearch:

    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2

    container_name: elasticsearch

    environment:

      - discovery.type=single-node

      - ELASTIC_PASSWORD=123456

      - bootstrap.memory_lock=true

      - ES_JAVA_OPTS=-Xms1g -Xmx1g

    ulimits:

      memlock:

        soft: -1

        hard: -1

    ports:

      - "9200:9200"

      - "9300:9300"

    volumes:

      - ./elasticsearch/data:/usr/share/elasticsearch/data

      - ./elasticsearch/logs:/usr/share/elasticsearch/logs

      - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro

    networks:

      - es-network

  kibana:

    image: docker.elastic.co/kibana/kibana:7.10.2

    container_name: kibana

    environment:

      - SERVER_PORT=5601

      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200

      - ELASTICSEARCH_USERNAME=elastic

      - ELASTICSEARCH_PASSWORD=123456

    ports:

      - "5601:5601"

    volumes:

      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro

      - ./kibana/logs:/usr/share/kibana/logs

    depends_on:

      - elasticsearch

    networks:

      - es-network

  metricbeat:

    image: docker.elastic.co/beats/metricbeat:7.10.2

    container_name: metricbeat

    user: root

    depends_on:

      - elasticsearch

    cap_add:

      - SYS_PTRACE

      - DAC_READ_SEARCH

    volumes:

      - ./metricbeat/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro

      - ./metricbeat/modules.d:/usr/share/metricbeat/modules.d:ro

      - /proc:/hostfs/proc:ro

      - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro

      - /:/hostfs:ro

    networks:

      - es-network

volumes: {}

networks:

  es-network:

    driver: bridge

1

2

3

4

5

6

cd /www/es-kibana

docker-compose down -v

docker-compose up -d

docker-compose logs -f elasticsearch

docker-compose logs -f kibana

docker-compose logs -f metricbeat

1

2

3

4

5

6

7

8

/www/es-kibana/

├── docker-compose.yml

├── elasticsearch/

│   └── elasticsearch.yml

├── kibana/

│   └── kibana.yml

├── data/             # Elasticsearch 数据目录（挂载）

└── logs/             # Elasticsearch 日志目录（挂载）

1

2

3

4

5

6

7

curl http://localhost:9200

#外网

curl http://127.0.0.1:9200

#kibana 获取密码

docker exec -it elasticsearch bin/elasticsearch-setup-passwords auto

elastic

123456

1

mkdir -p /www/filebeat/logs && cd /www/filebeat/logs

1

2

3

4

5

6

7

# 修改模板参数值 上传的参数不一致

setup.template.priority

# json解析问题调整

json.keys_under_root: true  # 修改这一行

json.add_error_key: true

json.message_key: json  # 修改这一行

# 先调试->在调试docker启动是否正常同步->启动镜像->启动正式容器

1

2

3

4

5

6

mkdir -p /www/filebeat/

mkdir -p /www/filebeat/modules.d

/www/filebeat/

├── docker-compose.yml

├── Dockerfile

└── filebeat.docker.yml

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

filebeat.config:

  modules:

    path: ${path.config}/modules.d/*.yml

    reload.enabled: false

filebeat.inputs:

  - type: log

    enabled: true

    paths:

      - /var/log/v99mian/**/*.log

      - /var/log/nginx/**/*.log

    json.keys_under_root: true

    json.add_error_key: true

    json.overwrite_keys: true

    fields:

      log_source: mian

processors:

  - decode_json_fields:

      fields: ["message"]

      target: ""

      overwrite_keys: true

  - timestamp:

      field: "@timestamp"

      layouts:

        - '2006-01-02T15:04:05.000Z07:00'

      timezone: "UTC"

  - add_host_metadata: {}

  - add_cloud_metadata: {}

  - add_docker_metadata: {}

  - add_kubernetes_metadata: {}

output.elasticsearch:

  hosts: ["127.0.0.1:9200"]

  username: "elastic"

  password: "123456"

  ssl.verification_mode: "none"

setup.template.name: "metricbeat-mian-prd"

setup.template.pattern: "metricbeat-*"

setup.template.priority: 260

setup.ilm.enabled: true

setup.ilm.rollover_alias: "metricbeat-mian-prd"

setup.ilm.pattern: "{now/d}-000001"

setup.ilm.policy_name: "metricbeat-mian-prd-policy"

setup.ilm.policy:

  policy:

    phases:

      hot:

        actions:

          rollover:

            max_age: "1d"

            max_size: "50gb"

      delete:

        min_age: "30d"

        actions:

          delete: {}

setup.template.settings:

  index.mapping.total_fields.limit: 2000

  index.mapping.ignore_malformed: true

  index.number_of_shards: 1

  index.number_of_replicas: 0

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

FROM docker.elastic.co/beats/filebeat:7.10.2

# 切换到 root（确保有权限修改配置文件属主）

USER root

# 复制配置文件到镜像中

COPY filebeat.docker.yml /usr/share/filebeat/filebeat.yml

# 如果 modules.d 目录下有自定义模块，也一并复制

COPY modules.d /usr/share/filebeat/modules.d

# 确保 filebeat 用户可以读取配置

RUN chown -R root:filebeat /usr/share/filebeat/filebeat.yml \

 && chmod 0644 /usr/share/filebeat/filebeat.yml

# 切回非 root 用户

USER filebeat

# 挂载日志目录

VOLUME ["/var/log/mian"]

VOLUME ["/var/log/nginx"]

# 启动命令

CMD ["filebeat", "-e", "--strict.perms=false", "-c", "/usr/share/filebeat/filebeat.yml"]

1

2

3

4

5

6

7

8

9

10

11

12

13

version: '3.8'

services:

  filebeat:

    build:

      context: .

      dockerfile: Dockerfile

    container_name: filebeat-mian

    restart: always

    user: root

    volumes:

      - /var/log/v99mian:/var/log/v99mian:ro

      - /var/log/nginx:/var/log/nginx:ro

      - /var/run/docker.sock:/var/run/docker.sock:ro

1

2

3

4

5

6

cd /www/filebeat

docker-compose down -v

docker-compose up -d

docker-compose up --build -d #调试启动

docker ps         # 查看容器运行状态

docker logs -f filebeat-mian   # 实时查看输出日志

1

2

3

4

curl -u elastic:123456 \

  'http://127.0.0.1:9200/metricbeat-v99mian-prd-*/_search?size=5&pretty'

curl -u elastic:123456 'http://127.0.0.1:9200/_cluster/health?pretty'

curl -u elastic:123456 'http://127.0.0.1:9200/_cat/indices?v'