香橙派4和树莓派4B构建K8S集群实践之四：BuildKit与LNMP网站首页 技术交流

# 生成base64编码的auth值, 其中user, pw为hub.docker.com的账密

echo -n "{user}:{pw}" | base64

vi ~/.docker/config.json
# example:
{
  "auths": {
          "https://index.docker.io/v1/": {
               "auth": "{auth}"
          }
  }
}

2.2 安装BuildKit

法一：在指定主机上安装使用

# amd64用这个
wget https://github.com/moby/buildkit/releases/download/v0.9.0/buildkit-v0.9.0.linux-amd64.tar.gz
tar xvf buildkit-v0.9.0.linux-amd64.tar.gz -C /usr/local/bin/

# arm64用这个
wget https://github.com/moby/buildkit/releases/download/v0.9.0/buildkit-v0.9.0.linux-arm64.tar.gz
tar xvf buildkit-v0.9.0.linux-arm64.tar.gz -C /usr/local/bin/

# 创建服务配置
cat >/etc/systemd/system/buildkitd.service <<EOF
[Unit]
Description=BuildKit
Documentation=https://github.com/moby/buildkit
After=containerd.service

[Service]
Type=notify
ExecStart=/usr/local/bin/buildkitd  --oci-worker=false --containerd-worker=true 

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl restart buildkitd 
systemctl enable buildkitd
systemctl status buildkitd

法2：创建 buildkitd和buildkit-cli pods，并在buildkit-cli里调用服务构建，需要说明的是: 我尚未能成功构建并推送上去仓库，可参考这篇文章Building Docker Images with BuildKit | Kubernetes Course Labs

# 安装buildkit server
kubectl apply -f https://kubernetes.courselabs.co/labs/buildkit/specs/buildkitd/buildkitd.yaml
kubectl logs -l app=buildkitd
# 删除buildkit
kubectl delete deploy,svc,pod -l kubernetes.courselabs.co=buildkit

kubectl create secret docker-registry registry-creds --docker-server=docker.io --docker-username={user} --docker-password={pw}
kubectl create configmap build-config --from-literal=REGISTRY=docker.io  --from-literal=REPOSITORY=bennybi

kubectl apply -f https://kubernetes.courselabs.co/labs/buildkit/specs/buildkit-cli/buildkit-cli.yaml

# 进入buildkit cli pod
kubectl exec -it buildkit-cli -- sh

# 在buildkit-cli pod内调用
buildctl --addr tcp://buildkitd:1234 build --frontend=dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${REGISTRY}/${REPOSITORY}/simple,push=true

3. 安装步骤

3.1 申请一个pvc存储区 (wwwroot-pvc.yaml)

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: wwwroot
  namespace: iot-age
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2G

3.2 Nginx

文件清单: nginx.yaml, nginx-configmap.yaml

iot-age中所有的nginx pods将使用统一的配置，ConfigMap (nginx-configmap.yaml)

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-conf
  namespace: iot-age
data:
  default.conf: |
    server {
        listen       80;
        listen  [::]:80;
        server_name  localhost;
        
        index index.php index.html;
        error_log  /var/log/nginx/error.log;
        access_log /var/log/nginx/access.log;
        root /var/www/html;
        location / {
            try_files $uri $uri/ /index.php?$query_string;
        }
        
        location ~ .php$ {
            try_files $uri =404;
            fastcgi_split_path_info ^(.+.php)(/.+)$;
            fastcgi_pass   php:9000;
            fastcgi_index  index.php;
            include        fastcgi_params;
            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
            fastcgi_param PATH_INFO $fastcgi_path_info;
        }

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }
    }

kubectl apply -f nginx-configmap.yaml

nginx.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nginx
  namespace: iot-age
spec:
  selector:
    matchLabels:
      app: nginx
  serviceName: "nginx"
  replicas: 2
  #volumeClaimTemplates:
  #  - kind: PersistentVolumeClaim
  #    apiVersion: v1
  #    metadata:
  #      name: wwwroot
  #    spec:
  #      accessModes:
  #        - ReadWriteOnce
  #      resources:
  #        requests:
  #          storage: 2Gi
  #      volumeMode: Filesystem
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 80
        volumeMounts:
        - name: wwwroot
          mountPath: /var/www/html
        - name: nginx-conf
          mountPath: /etc/nginx/conf.d/
      volumes:
      - name: wwwroot
        persistentVolumeClaim:
          claimName: wwwroot
      - name: nginx-conf
        configMap:
         name: nginx-conf
status: {}

---
apiVersion: v1
kind: Service
metadata:
  name: nginx
  namespace: iot-age
spec:
  type: NodePort
  ports:
  - name: nginx
    port: 80
    protocol: TCP
    targetPort: 80
    nodePort: 30010
  selector:
    app: nginx

kubectl apply -f nginx.yaml

3.3 php-fpm

文件清单: Dockerfile, php.yaml

Dockerfile，该文件参考了php/Dockerfile at master · khs1994-docker/php · GitHub

# syntax=docker/dockerfile:labs
ARG PHP_VERSION=8.2.5

FROM --platform=$TARGETPLATFORM php:${PHP_VERSION}-fpm-alpine3.17 as php

LABEL maintainer="khs1994-docker/lnmp <khs1994@khs1994.com>"

ARG PHP_EXTENSION_EXTRA

ARG PECL_EXTENSION_EXTRA

ARG APK_EXTRA

ARG APK_DEV_EXTRA

ENV TZ=Asia/Shanghai 
    APP_ENV=development

ENV PHP_EXTENSION 
      bcmath 
      bz2 
      calendar 
      enchant 
      exif 
      gd 
      gettext 
      gmp 
      imap 
      intl 
      mysqli 
      pcntl 
      pdo_pgsql 
      pdo_mysql 
      pgsql 
      sockets 
      sysvmsg 
      sysvsem 
      sysvshm 
      # tidy 
      # xsl 
      zip 
      shmop 
      ffi 
      ${PHP_EXTENSION_EXTRA:-}

ENV PECL_EXTENSION 
      # mongodb 
      # https://github.com/mongodb/mongo-php-driver/archive/master.tar.gz 
      # igbinary 
      https://github.com/igbinary/igbinary/archive/master.tar.gz 
      # redis 
      https://github.com/phpredis/phpredis/archive/develop.tar.gz 
      memcached 
# 安装测试版的扩展，可以在扩展名后加 -beta
      # yaml 
      # https://github.com/php/pecl-file_formats-yaml/archive/php7.tar.gz 
      https://github.com/kjdev/php-ext-zstd/archive/0.11.0.tar.gz 
      ${PECL_EXTENSION_EXTRA:-}

ARG ALPINE_URL=dl-cdn.alpinelinux.org

RUN sed -i "s/dl-cdn.alpinelinux.org/${ALPINE_URL}/g" /etc/apk/repositories 
      && set -xe 
      
      # && apk add --no-cache patch 
      # && cd / 
      # && { 
      #   echo "--- a/usr/local/lib/php/build/php.m4"; 
      #    } | tee 1.diff 
      # && patch -p1 < 1.diff 
      # && apk del --no-network patch 
      # && rm -rf 1.diff 
      
# 不要删除
      && PHP_FPM_RUN_DEPS=" 
                         bash 
                         tzdata 
                         libmemcached-libs 
                         libpq 
                         libzip 
                         zlib 
                         libpng 
                         freetype 
                         libjpeg-turbo 
                         libxpm 
                         libwebp 
                         libbz2 
                         libexif 
                         gmp 
                         enchant2 
                         c-client 
                         icu-libs 
                         zstd-libs 
                         linux-headers 
                         libavif 
                         ${APK_EXTRA:-} 
                         " 
                         # tidyhtml-libs 
                         # libxslt 
                         # yaml 
# *-dev 编译之后删除
      && PHP_FPM_BUILD_DEPS=" 
                         openssl-dev 
                         libmemcached-dev 
                         cyrus-sasl-dev 
                         postgresql-dev 
                         libzip-dev 
                         zlib-dev 
                         libpng-dev 
                         freetype-dev 
                         libjpeg-turbo-dev 
                         libxpm-dev 
                         libwebp-dev 
                         libexif-dev 
                         gmp-dev 
                         bzip2-dev 
                         enchant2-dev 
                         imap-dev 
                         gettext-dev 
                         libwebp-dev 
                         icu-dev 
                         zstd-dev 
                         libavif-dev 
                         ${APK_DEV_EXTRA:-} 
                         " 
                         # tidyhtml-dev 
                         # libxslt-dev 
                         # yaml-dev 
        && apk add --no-cache --virtual .php-fpm-run-deps $PHP_FPM_RUN_DEPS 
        && apk add --no-cache --virtual .php-fpm-build-deps $PHP_FPM_BUILD_DEPS 
        && apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS 
        && curl -fsSL -o /usr/local/bin/pickle 
           https://github.com/khs1994-php/pickle/releases/download/nightly/pickle-debug.phar 
        && chmod +x /usr/local/bin/pickle 
# 安装内置扩展
        && docker-php-source extract 
        && docker-php-ext-configure zip 
                                    --with-zip 
        && docker-php-ext-install zip 
        && strip --strip-all $(php-config --extension-dir)/zip.so 
        # && docker-php-ext-configure gd 
        && echo " 
                                        --disable-gd-jis-conv 
                                        --with-freetype 
                                        --with-jpeg 
                                        --with-webp 
                                        --with-xpm 
                                        --with-avif" > /tmp/gd.configure.options 
        # && docker-php-ext-install $PHP_EXTENSION 
        && pickle install $PHP_EXTENSION -n --defaults --strip 
        && docker-php-source delete 
# 安装 PECL 扩展
        && echo "--enable-redis-igbinary --enable-redis-zstd" > /tmp/redis.configure.options 
        && echo "--enable-memcached-igbinary" > /tmp/memcached.configure.options 
        && echo "--with-libzstd" > /tmp/zstd.configure.options 
        && pickle install $PECL_EXTENSION -n --defaults 
           --strip --cleanup 
# 默认不启用的扩展
        && pickle install 
# https://github.com/tideways/php-xhprof-extension.git
             https://github.com/tideways/php-xhprof-extension/archive/master.tar.gz 
             -n --defaults --strip --cleanup --no-write 
        && pickle install opcache 
        # && docker-php-ext-enable opcache 
        && apk del --no-network .phpize-deps .php-fpm-build-deps 
        && rm -rf /tmp/* 
# 创建日志文件夹
        && mkdir -p /var/log/php-fpm 
        && ln -sf /dev/stdout /var/log/php-fpm/access.log 
        && ln -sf /dev/stderr /var/log/php-fpm/error.log 
        && chmod -R 777 /var/log/php-fpm 
        && rm -rf /usr/local/lib/php/.registry/.channel.pecl.php.net/* 
        && php -m 
        && ls -la $(php-config --extension-dir) 
        && php -d error_reporting=22527 -d display_errors=1 -r 'var_dump(iconv("UTF-8", "UTF-8//IGNORE", "This is the Euro symbol '''€'''."));' 
# smoke test
        && php --version

WORKDIR /app

ARG VCS_REF="unknow"

LABEL org.opencontainers.image.revision=$VCS_REF 
      org.opencontainers.image.source="https://github.com/khs1994-docker/php"

3.3.1 构建并推送镜像

buildctl build
--frontend=dockerfile.v0
--local context=.
--local dockerfile=.
--export-cache type=inline
--output type=image,name=docker.io/bennybi/php-fpm:v1,push=true

php.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: php
  namespace: iot-age
spec:
  selector:
    matchLabels:
      app: php
      tier: backend
  replicas: 2
  template:
    metadata:
      labels:
        app: php
        tier: backend
    spec:
      containers:
      - name: php
        #image: php:8.2-fpm
        image: bennybi/php-fpm:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9000
        volumeMounts:
        - name: wwwroot
          mountPath: /var/www/html/
 
      volumes:
      - name: wwwroot
        persistentVolumeClaim:
          claimName: wwwroot
 
---
apiVersion: v1
kind: Service
metadata:
  name: php
  namespace: iot-age
spec:
  ports:
  - name: php
    port: 9000
    protocol: TCP
    targetPort: 9000
  selector:
    app: php