开源版JuiceFS on Amazon EKS 上的实践

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "AllowS3Access",
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket",
                "s3:PutObject",
                "s3:GetObject",
                "s3:DeleteObject",
                "s3:ListMultipartUploadParts",
                "s3:AbortMultipartUpload"
            ],
            "Resource": [
                "arn:aws:s3:::juicefs-demo-eks",
                "arn:aws:s3:::juicefs-demo-eks/*"
            ]
        },
        {
            "Sid": "AllowValkeyAccess",
            "Effect": "Allow",
            "Action": [
                "elasticache:Describe*",
                "elasticache:ListTagsForResource"
            ],
            "Resource": "*"
        }
    ]
}

# 默认安装到 /usr/local/bin
sudo curl -sSL https://d.juicefs.com/install | sh -

# 确认安装的版本
[ec2-user@ip-172-31-27-182 bin]$ juicefs -V
juicefs version 1.3.0+2025-07-03.30190ca

[ssm-user@ip-172-31-66-201 ~]$ juicefs format --storage s3  --bucket https://s3.ap-northeast-2.amazonaws.com/juicefs-demo-eks  rediss://default:juicefspasswordabc@clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379/1 juicefs-apn2

[ssm-user@ip-172-31-66-201 ~]$ sudo /usr/local/bin/juicefs mount -d rediss://default:juicefspasswordabc@clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379/1 /mnt/juicefs
2025/08/04 09:43:25.707487 juicefs[604981] <INFO>: Meta address: rediss://default:****@clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379/1 [NewClient@interface.go:578]
2025/08/04 09:43:25.724902 juicefs[604981] <INFO>: redis clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379 is in cluster mode [newRedisMeta@redis.go:214]
2025/08/04 09:43:25.737225 juicefs[604981] <WARNING>: AOF is not enabled, you may lose data if Redis is not shutdown properly. [checkRedisInfo@info.go:84]
2025/08/04 09:43:25.737951 juicefs[604981] <INFO>: Ping redis latency: 619.556µs [checkServerConfig@redis.go:3692]
2025/08/04 09:43:25.739458 juicefs[604981] <INFO>: Data use s3://juicefs-demo-eks/juicefs-apn2/ [mount@mount.go:588]
2025/08/04 09:43:26.241453 juicefs[604981] <INFO>: OK, juicefs-apn2 is ready at /mnt/juicefs [checkMountpoint@mount_unix.go:235]

# 将 juicefs 客户端重命名为 mount.juicefs 并复制到 /sbin/ 目录
[root@ip-172-31-65-22 ~]# cp /usr/local/bin/juicefs /sbin/mount.juicefs
# 编辑/etc/fstab配置文件，添加下边的新记录
rediss://default:juicefspasswordabc@clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379/1    /mnt/juicefs       juicefs     _netdev,cache-size=20480     0  0

juicefs bench ~/jfs

sudo yum install -y gcc jemalloc-devel openssl-devel tcl tcl-devel make
# 下载valkey-cli的源码包（以最新版本为例，这里用的是8.1.3版本）：

wget https://github.com/valkey-io/valkey/archive/refs/tags/8.1.3.tar.gz
# 解压源码包：

tar xvzf 8.1.3.tar.gz
cd valkey-8.1.3/
# 编译valkey-cli：
如果是第一次编译，建议先执行清理：

make distclean
# 然后带TLS支持编译valkey-cli：

make valkey-cli BUILD_TLS=yes
# 编译成功后，执行安装

sudo make install
# 安装完成后，可以用以下命令验证valkey-cli是否可用：

valkey-cli –version

[ssm-user@ip-172-31-27-182 ec2-user]$ valkey-cli --tls -h clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com -p 6379 -a juicefspasswordabc
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.

clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379> KEYS *

apiVersion: v1
kind: Pod
metadata:
  name: juicefs-hostpath
spec:
  containers:
  - name: juicefs-app
    image: busybox
    command: ["/bin/sh", "-c", "ls /mnt/juicefs && sleep 3600"]   # 列出挂载目录内容后休眠，方便调试查看
    volumeMounts:
    - name: juicefs-volume
      mountPath: /mnt/juicefs
  volumes:
  - name: juicefs-volume
    hostPath:
      path: /mnt/juicefs/   # JuiceFS 在节点上的挂载目录
      type: Directory

kubectl get nodes # 确认节点状态

kubectl apply -f juicefs-hostpath.yaml # 创建pod

kubectl get pod juicefs-hostpath-example # 检查 Pod 状态
[ec2-user@ip-172-31-27-182 ~]$ kubectl get pod
NAME                          READY   STATUS    RESTARTS   AGE
backend-655f66b77-94jds       1/1     Running   0          238d
frontend-64db969c4c-wfstf     1/1     Running   0          238d
juicefs-hostpathe             1/1     Running   0          20s
php-apache-77b9d477b8-9t7qn   1/1     Running   0          236d


kubectl exec -it juicefs-hostpath-example -- sh # 进入容器验证
ls -l /mnt/juicefs

[ec2-user@ip-172-31-27-182 ~]$ kubectl exec -it juicefs-hostpath -- sh
/ # ls -l /mnt/juicefs
total 0
-rw-rw-r--    1 1001     1001             0 Aug  4 14:58 test.txt

[ec2-user@ip-172-31-27-182 bin]$ aws eks describe-cluster --name three-tier-cluster --query "cluster.identity.oidc.issuer" --output text
https://oidc.eks.ap-northeast-2.amazonaws.com/id/DBBBA5D9723CBD0418D0301B27886659

# 附加“环境准备”章节的安全策略juicefs-policy（替换YOUR_POLICY_ARN）
eksctl create iamserviceaccount \
--name juicefs-csi-controller-sa \
--namespace kube-system \
--cluster three-tier-cluster \
--attach-policy-arn arn:aws:iam::1234567890:policy/juicefs-policy \
--override-existing-serviceaccounts \
--approve

# 检查已附加的策略
aws iam list-attached-role-policies --role-name JuiceFS-CSI-Driver-Role

curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

helm repo add juicefs https://charts.juicefs.com
helm repo update

kubeletDir: "/var/lib/kubelet"

mountMode: "process"   # 这里为 process 模式

image:
  repository: "juicedata/juicefs-csi-driver"
  tag: "v0.26.4"

webhook:
  certManager:
    enabled: false

provisioner:
  enabled: true       # 开启动态 Provisioner, 支持自动创建 PV 和挂载 JuiceFS 子目录

controller:
  resources:
    limits:
      cpu: 1000m
      memory: 512Mi
    requests:
      cpu: 200m
      memory: 256Mi

node:
  resources:
    limits:
      cpu: 500m
      memory: 256Mi
    requests:
      cpu: 100m
      memory: 128Mi

helm upgrade --install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f values.yaml

apiVersion: v1
kind: Secret
metadata:
  name: juicefs-secret
  namespace: kube-system
type: Opaque
stringData:
  name: juicefs-apn2  #之前创建的juicefs
  metaurl: "rediss://default:juicefspasswordabc@clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379/1"
  storage: s3
  bucket: "https://s3.ap-northeast-2.amazonaws.com/juicefs-demo-eks"
  access-key: ""
  secret-key: ""

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: juicefs-csi
provisioner: csi.juicefs.com
parameters:
  # 引用 Secret
  csi.storage.k8s.io/provisioner-secret-name: juicefs-secret
  csi.storage.k8s.io/provisioner-secret-namespace: kube-system
  csi.storage.k8s.io/node-publish-secret-name: juicefs-secret
  csi.storage.k8s.io/node-publish-secret-namespace: kube-system
reclaimPolicy: Delete
volumeBindingMode: Immediate

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: juicefs-pvc
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
  storageClassName: juicefs-csi

apiVersion: v1
kind: Pod
metadata:
  name: juicefs-test
spec:
  containers:
  - name: app
    image: alpine:latest  # 轻量级测试镜像
    command: ["sh", "-c", "echo 'JuiceFS Test' > /data/juicefs.txt && sleep 3600"]
    volumeMounts:
    - name: juicefs-pvc
      mountPath: /data  # 挂载路径
  volumes:
  - name: juicefs-pvc
    persistentVolumeClaim:
      claimName: juicefs-pvc  # 必须与当前的PVC名称一致

# 查看 CSI Driver Pod 运行状态：
kubectl get pods -n kube-system -l app=juicefs-csi-driver

#创建 Pod 并进入容器，测试读写挂载目录：
kubectl exec -it juicefs-app -- sh
touch /data/testfile
ls -l /data/testfile

FROM ubuntu:22.04

# 设置JuiceFS版本
ARG JUICEFS_VERSION=1.3.0

# 安装依赖
RUN apt-get update && apt-get install -y \
    ca-certificates \
    curl \
    fuse3 \
    && rm -rf /var/lib/apt/lists/*

# 下载并安装JuiceFS
RUN set -eux; \
    ARCH=$(uname -m); \
    case "${ARCH}" in \
        x86_64) ARCH=amd64 ;; \
        aarch64) ARCH=arm64 ;; \
        *) echo "unsupported architecture: ${ARCH}"; exit 1 ;; \
    esac; \
    curl -fSL "https://github.com/juicedata/juicefs/releases/download/v${JUICEFS_VERSION}/juicefs-${JUICEFS_VERSION}-linux-${ARCH}.tar.gz" -o juicefs.tar.gz; \
    tar -xzf juicefs.tar.gz; \
    install juicefs /usr/local/bin/juicefs; \
    rm juicefs.tar.gz juicefs; \
    juicefs version

# 创建用户和目录
RUN groupadd -r juicefs && useradd -r -g juicefs juicefs
RUN mkdir -p /var/juicefs && chown juicefs:juicefs /var/juicefs

WORKDIR /var/juicefs
USER juicefs

EXPOSE 9000

# 设置入口点，支持自定义参数
ENTRYPOINT ["juicefs"]
CMD ["gateway", "--help"]

#!/bin/bash

# JuiceFS Gateway镜像构建脚本
set -e

# 配置变量
IMAGE_NAME="juicefs-gateway"
TAG="latest"
JUICEFS_VERSION="1.3.0"

echo "开始构建JuiceFS Gateway镜像..."
echo "JuiceFS版本: v$JUICEFS_VERSION"

# 构建镜像
docker build \
    --build-arg JUICEFS_VERSION=$JUICEFS_VERSION \
    -t $IMAGE_NAME:$TAG .

echo "镜像构建完成: $IMAGE_NAME:$TAG"

# 测试镜像基本功能
echo "测试镜像功能..."
docker run --rm $IMAGE_NAME:$TAG version
docker run --rm $IMAGE_NAME:$TAG gateway --help | head -5

echo "构建和测试完成！"

set -e

# 配置变量
IMAGE_NAME="juicefs-gateway"
TAG="latest"
AWS_ACCOUNT_ID="1234567890"  # 替换AWS账户ID
AWS_REGION="ap-northeast-2"           # 根据EKS区域调整
ECR_REPO_NAME="juicefs-gateway"

echo "登录到AWS ECR..."
# 获取ECR登录密码并登录
aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com

echo "检查ECR仓库..."
# 检查仓库是否存在，如果不存在则创建
if ! aws ecr describe-repositories --repository-names $ECR_REPO_NAME --region $AWS_REGION > /dev/null 2>&1; then
    echo "创建ECR仓库: $ECR_REPO_NAME"
    aws ecr create-repository \
        --repository-name $ECR_REPO_NAME \
        --region $AWS_REGION \
        --image-scanning-configuration scanOnPush=true \
        --image-tag-mutability MUTABLE
fi

echo "标记镜像..."
# 标记镜像用于ECR
docker tag $IMAGE_NAME:$TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPO_NAME:$TAG

echo "推送镜像到ECR..."
# 推送镜像到ECR
docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPO_NAME:$TAG

echo "镜像推送完成!"
echo "ECR镜像地址: $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPO_NAME:$TAG"

# 显示仓库信息
echo "ECR仓库信息:"
aws ecr describe-repositories --repository-names $ECR_REPO_NAME --region $AWS_REGION --query 'repositories[0].repositoryUri' --output text

apiVersion: apps/v1
kind: Deployment
metadata:
  name: juicefs-s3-gateway
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: juicefs-s3-gateway
  template:
    metadata:
      labels:
        app: juicefs-s3-gateway
    spec:
      containers:
      - name: juicefs-s3-gateway
        image: 1234567890.dkr.ecr.ap-northeast-2.amazonaws.com/juicefs-gateway:latest # 指定ECR的地址
        env:
        - name: MINIO_ROOT_USER
          value: "admin"
        - name: MINIO_ROOT_PASSWORD
          value: "password"
        command: ["juicefs"]
        args:
          - "gateway"
          - "rediss://default:juicefspassword@clustercfg.juicefs-valkey.kjmaj8.apn2.cache.amazonaws.com:6379/1" # 指定valkey访问链接的用户名和口令
          - "0.0.0.0:9000"
        ports:
        - containerPort: 9000

apiVersion: v1
kind: Service
metadata:
  name: juicefs-s3-gateway
  namespace: kube-system
spec:
  selector:
    app: juicefs-s3-gateway
  ports:
  - port: 9000
    targetPort: 9000
    protocol: TCP
  type: NodePort     # ALB 需要用 NodePort 或 LoadBalancer 类型

apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
  name: alb
  annotations:
    ingressclass.kubernetes.io/is-default-class: "true"  # 设置默认 ingressclass（非必需）
spec:
  controller: eks.amazonaws.com/alb
  parameters:
    apiGroup: eks.amazonaws.com
    kind: IngressClassParams

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: juicefs-s3-gateway-ingress
  namespace: kube-system
  annotations:
    kubernetes.io/ingress.class: alb
spec:
  rules:
  - http:
      paths:
      - path: /*
        pathType: ImplementationSpecific
        backend:
          service:
            name: juicefs-s3-gateway
            port:
              number: 9000

kubectl get ingress -n kube-system juicefs-s3-gateway-ingress -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'