iLogtail：日志收集 Clickhouse：日志存储 Metabase：可视化

安装 ClickHouse

docker run -d --restart=always --name clickhouse-server \
--ulimit nofile=262144:262144 \
-p 8123:8123 \
-p 8999:9000 \
-v /data/clickhouse/data:/var/lib/clickhouse \
-v /data/clickhouse/logs:/var/log/clickhouse-server \
-v /etc/localtime:/etc/localtime:ro \
-e CLICKHOUSE_USER=root \
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
-e CLICKHOUSE_PASSWORD=ck_adm123 \
clickhouse/clickhouse-server:23.8.4.69

日志收集 iLogtail

cd /data
wget https://github.com/starriesWEB/ilogtail/releases/download/1.8/ilogtail.tar.gz
tar -xvf ilogtail.tar.gz
cd ilogtail

配置文件目录为 user_yaml_config.d

[root@localhost user_yaml_config.d]## cat docker_log.yaml 
enable: true
inputs:
  - Type: service_docker_stdout
    Stdout: true
    Stderr: true
    BeginLineCheckLength: 10
    BeginLineRegex: \d+-\d+-\d+.*(INFO|ERROR|DEBUG|WARN).*
    IncludeEnv:
      JAVA_VERSION: "jdk8u262-b10_openj9-0.21.0"
processors:
  - Type: processor_split_log_regex
    SplitRegex: \d+-\d+-\d+.*
    SplitKey: content
    PreserveOthers: true
  - Type: processor_drop
    DropKeys:
      - "_time_"
      - "_source_"
      - "_image_name_"
      - "_container_ip_"
flushers:
  - Type: flusher_clickhouse
    Addresses: ["127.0.0.1:8999"]
    Authentication:
      PlainText:
        Database: default
        Username: root
        Password: ck_adm123
      TLS:
        Enabled: false
    Table: blade
      #- Type: flusher_stdoutdd
      #FileName: /root/output/out_multiline.log
      #OnlyStdout: false
    ##  - Type: flusher_stdout
    #OnlyStdout: true

配置守护进程

cat>/etc/systemd/system/ilogtaild.service<<EOF
[Unit]
Description=ilogtail

[Service]
Type=simple
User=root
Restart=always
RestartSec=60
ExecStart=/data/ilogtail/ilogtail

[Install]
WantedBy=multi-user.target
EOF

注册服务

systemctl daemon-reload

启动服务

systemctl restart ilogtaild

查看服务状态

systemctl status ilogtaild

启动成功后会在 clickhouse 中自动创建 2 个表 ilogtail_blade 和 ilogtail_blade_buffer

创建表

创建日志表

CREATE TABLE default.blade
(
    `logTime`       DATETIME,
    `hostName`      String,
    `hostIp`        String,
    `containerName` String,
    `content`       String
)
    ENGINE = MergeTree PARTITION BY toYYYYMMDD(logTime)
        ORDER BY (logTime, containerName)
		TTL logTime + INTERVAL 7 DAY
        SETTINGS index_granularity = 8192;

创建物化事务处理并传输到日志表

CREATE MATERIALIZED VIEW default.blade_view
            TO default.blade
            (
             `logTime` DATETIME,
             `hostName` String,
             `hostIp` String,
             `containerName` String,
             `content` String
                )
AS
SELECT
    toDateTime(_timestamp) AS logTime,
    hostName,
    hostIp,
    containerName,
    content
FROM
    default.ilogtail_blade;

Metabase 数据可视化

mkdir -p /data/metabase/plugins
cd /data/metabase/plugins
## 下载 clickhouse 驱动
wget https://github.com/ClickHouse/metabase-clickhouse-driver/releases/download/1.2.2/clickhouse.metabase-driver.jar
chmod -R 777 /data/metabase

启动 Metabase

docker run -d --restart=always -p 3000:3000 \
-v /data/metabase/data:/metabase-data \
-v /data/metabase/plugins:/plugins \
-e "MB_DB_FILE=/metabase-data/metabase.db" \
--name metabase metabase/metabase:v0.47.0

添加数据源，查看数据