您现在的位置是：首页 >技术杂谈 >fluent-bit日志上传到s3网站首页技术杂谈

s3存储桶设置权限：

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Principal": "*",
            "Sid": "AllowS3Self",
            "Effect": "Allow",
            "Action": "s3:*", 
            "Resource": [
                "arn:aws:s3:::firelens-access-log/*", 
                "arn:aws:s3:::firelens-access-log" 
            ]
        }
       
    ]
}
 

{
    "family": "firelens-sample-s3-ec2-demo",
    "taskRoleArn": "arn:aws:iam::254278701124:role/ecsTaskExecutionRole",
    "executionRoleArn": "arn:aws:iam::254278701124:role/ecsTaskExecutionRole",
    "containerDefinitions": [
        {
            "essential": true,
            "name": "log_router",
            "image": "402097323/fluent-bit-s3:v2",
            "logConfiguration": {
                "logDriver": "awslogs",
                "options": {
                    "awslogs-create-group": "true",
                    "awslogs-group": "/ecs/firelens-sample",
                    "awslogs-region": "us-east-1",
                    "awslogs-stream-prefix": "ecs"
                }
            },
            "mountPoints": [
                {
                    "sourceVolume": "data",
                    "containerPath": "/data",
                    "readOnly": false
                }
            ],
            "firelensConfiguration": {
                "type": "fluentbit",
                "options": {
                    "config-file-type": "file",
                    "config-file-value": "/extra.conf",
                    "enable-ecs-log-metadata": "false"
                }
            },
            "user": "0"
        },
        {
            "essential": true,
            "name": "myapp",
            "image": "402097323/applog:v2",
            "logConfiguration": {
                "logDriver": "awsfirelens"
            },
            "mountPoints": [
                {
                    "sourceVolume": "data",
                    "containerPath": "/data",
                    "readOnly": false
                }
            ],
            "dependsOn": [
                {
                    "containerName": "log_router",
                    "condition": "START"
                }
            ]
        }
    ],
    "cpu": "256",
    "memory": "512",
    "volumes": [
        {
            "name": "data",
            "host": {}
        }
    ],
    "compatibilities": [
    "EXTERNAL",
    "EC2"
  ],
     "networkMode": "awsvpc"
}

/extra.conf根据实际情况监控路径，402097323/fluent-bit-s3:v2 这个镜像要启动容器，然后把extra.conf文件复制到根路径下

docker run -itd --name ffff1 402097323/fluent-bit-s3:v2

docker cp 75bb7ae358a1:/extra.conf   extra.conf

docker commit 容器实例id  镜像名称

网络部署成桥接模式，否则可能会网络不通，在加上s3权限设置；任务类型别设置fargate

[SERVICE]
    Parsers_File /fluent-bit/parsers/parsers.conf
    Flush 1
    Grace 30

[INPUT]
    Name tail
    Path /data/logs/access.log
    Tag access

[INPUT]
    Name tail
    Path /data/logs/error.log
    Tag error

[OUTPUT]
    Name s3
    Match *
    bucket firelens-access-log
    region us-east-1
    upload_timeout  1m
    total_file_size 10M

docker logs 088fc8999f35 -f

[2023/05/09 12:09:50] [ info] [output:s3:s3.0] Successfully uploaded object /fluent-bit-logs/access/2023/05/09/12/08/40-objectfDHCcciD
[2023/05/09 12:09:50] [ info] [output:s3:s3.0] Successfully uploaded object /fluent-bit-logs/error/2023/05/09/12/08/40-objectBwI4KFtG
[2023/05/09 12:11:00] [ info] [output:s3:s3.0] Successfully uploaded object /fluent-bit-logs/access/2023/05/09/12/09/50-objectMQrxDKaJ