[单master节点k8s部署]32.ceph分布式存储（三）

回到 ceph 管理节点创建 pool 池：

[root@master1-admin ~]# ceph osd pool create k8stest 56
pool 'k8stest' created
[root@master1-admin ~]# rbd create rbda -s 1024 -p k8stest
[root@master1-admin ~]# rbd feature disable k8stest/rbda object-map fast-diff deepflatten

创建pv（RBD共享挂载）

[root@xianchaomaster1 ~]# cat pv.yaml 
apiVersion: v1 
kind: PersistentVolume 
metadata: name: ceph-pv 
spec: capacity: storage: 1Gi accessModes: - ReadWriteOnce rbd: monitors: - '192.168.40.201:6789'- '192.168.40.200:6789'- '192.168.40.202:6789' pool: k8stest image: rbda user: admin secretRef: name: ceph-secret fsType: xfs readOnly: false persistentVolumeReclaimPolicy: Recycle

ceph rbd 块存储的特点，ceph rbd 块存储能在同一个 node 上跨 pod 以 ReadWriteOnce 共享挂载，ceph rbd 块存储能在同一个 node 上同一个 pod 多个容器中以 ReadWriteOnce 共享挂载，ceph rbd 块存储不能跨 node 以 ReadWriteOnce 共享挂载。如果一个使用ceph rdb 的pod所在的node挂掉，这个pod虽然会被调度到其它node， 但是由于 rbd 不能跨 node 多次挂载和挂掉的 pod 不能自动解绑 pv 的问题，这个新 pod 不会正常运行。

storageClass动态生成pv

role和clusterrole等配置：

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: rbd-provisioner
rules:- apiGroups: [""]resources: ["persistentvolumes"]verbs: ["get", "list", "watch", "create", "delete"]- apiGroups: [""]resources: ["persistentvolumeclaims"]verbs: ["get", "list", "watch", "update"]- apiGroups: ["storage.k8s.io"]resources: ["storageclasses"]verbs: ["get", "list", "watch"]- apiGroups: [""]resources: ["events"]verbs: ["create", "update", "patch"]- apiGroups: [""]resources: ["services"]resourceNames: ["kube-dns","coredns"]verbs: ["list", "get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: rbd-provisioner
subjects:- kind: ServiceAccountname: rbd-provisionernamespace: default
roleRef:kind: ClusterRolename: rbd-provisionerapiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:name: rbd-provisioner
rules:
- apiGroups: [""]resources: ["secrets"]verbs: ["get"]
- apiGroups: [""]resources: ["endpoints"]verbs: ["get", "list", "watch", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:name: rbd-provisioner
roleRef:apiGroup: rbac.authorization.k8s.iokind: Rolename: rbd-provisioner
subjects:
- kind: ServiceAccountname: rbd-provisionernamespace: default

设置rbd provisioner

apiVersion: apps/v1
kind: Deployment
metadata:name: rbd-provisioner
spec:selector:matchLabels:app: rbd-provisionerreplicas: 1strategy:type: Recreatetemplate:metadata:labels:app: rbd-provisionerspec:containers:- name: rbd-provisionerimage: quay.io/xianchao/external_storage/rbd-provisioner:v1imagePullPolicy: IfNotPresentenv:- name: PROVISIONER_NAMEvalue: ceph.com/rbdserviceAccount: rbd-provisioner

设置secret和sa

apiVersion: v1
kind: ServiceAccount
metadata:name: rbd-provisioner

apiVersion: v1
kind: Secret
metadata:name: ceph-secret-1type: "ceph.com/rbd"data:key: QVFBWk0zeGdZdDlhQXhBQVZsS0poYzlQUlBianBGSWJVbDNBenc9PQ==

storageclass

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: k8s-rbdprovisioner: ceph.com/rbdparameters:monitors: 192.168.40.201:6789adminId: adminadminSecretName: ceph-secret-1pool: k8stest1userId: adminuserSecretName: ceph-secret-1fsType: xfsimageFormat: "2"imageFeatures: "layering

pvc

kind: PersistentVolumeClaim
apiVersion: v1
metadata:name: rbd-pvc
spec:accessModes:- ReadWriteOncevolumeMode: Filesystemresources:requests:storage: 1GistorageClassName: k8s-rbd

发现PVC绑定错误，绑定的为k8stest1,而创建的pool资源为k8stest。

创建新的pool资源：ceph osd pool create k8stest1 56

测试pod

[root@xianchaomaster1 ~]# cat pod-sto.yaml 
apiVersion: v1
kind: Pod
metadata:labels:test: rbd-podname: ceph-rbd-pod
spec:containers:- name: ceph-rbd-nginximage: nginximagePullPolicy: IfNotPresentvolumeMounts:- name: ceph-rbdmountPath: /mntreadOnly: falsevolumes:- name: ceph-rbdpersistentVolumeClaim:claimName: rbd-pvc

K8s挂载cephFS

k8s挂载ceph后，就可以将ceph作为一个存储路径使用。

仍使用之前创建的secret文件。

先在ceph集群的节点上创建一个secret file

[root@master1-admin ~]# cat /etc/ceph/ceph.client.admin.keyring |grep key|awk -
F" " '{print $3}' > /etc/ceph/admin.secret

然后在ceph集群中创建一个目录，并把ceph的根目录挂载到这个目录。xianchao作为一个文件系统，是我们之前已经创建好了的（见CSDN）。这里再重复一遍：

[root@master1-admin ceph]# ceph osd pool create cephfs_data 26
pool 'cephfs_data' created
[root@master1-admin ceph]# ceph osd pool create cephfs_metadata 26
pool 'cephfs_metadata' created
[root@master1-admin ceph]# ceph fs new xianchao cephfs_metadata cephfs_data

[root@master1-admin ~]# mkdir xianchao_data
[root@master1-admin ~]# mount -t ceph 192.168.40.201:6789:/ /root/xianchao_data 
-o name=admin,secretfile=/etc/ceph/admin.secret
[root@master1-admin ~]# df -h
192.168.40.201:6789:/ 165G 106M 165G 1% /root/xianchao_data
在 cephfs 的根目录里面创建了一个子目录 lucky，k8s 以后就可以挂载这个目录
[root@master1-admin ~]# cd /root/xianchao_data/
[root@master1-admin xianchao_data]# mkdir lucky
[root@master1-admin xianchao_data]# chmod 0777 lucky/

mount -t ceph 表明挂载的是一个cephFS系统，而网址是monitor的地址，跟着他的根目录。将这个根目录挂载到xianchao_data 文件夹下。

使用 admin 用户来挂载 CephFS。

使用secret 来验证这个挂载。

这里只有xianchao一个cephFS，如果有多个，可以指定cephFS的名称：

mount -t ceph 192.168.40.201:6789:/xianchao /root/xianchao_data -o name=admin,secretfile=/etc/ceph/admin.secret

CephFS 是一个分布式文件系统，允许将其挂载到不同节点上。为了更方便地管理和组织数据，通常会在 CephFS 挂载的根目录下创建子目录。例如，在这个的场景中，lucky 目录是为了以后 Kubernetes 的持久卷使用。

创建pv

[root@xianchaomaster1 ceph]# cat cephfs-pv.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:name: cephfs-pv
spec:capacity:storage: 1GiaccessModes:- ReadWriteManycephfs:monitors:- 192.168.40.201:6789path: /luckyuser: adminreadOnly: falsesecretRef:name: cephfs-secretpersistentVolumeReclaimPolicy: Recycle

其余的pvc创建和之前类似，通过创建在不同节点上的pod，可以看到不同node上的pod可以共享一个pv。

pod文件

[root@xianchaomaster1 ceph]# cat cephfs-pod-1.yaml 
apiVersion: v1
kind: Pod
metadata:name: cephfs-pod-1
spec:containers:- image: nginxname: nginximagePullPolicy: IfNotPresentvolumeMounts:- name: test-v1mountPath: /mntvolumes:- name: test-v1persistentVolumeClaim:claimName: cephfs-pvc
[root@xianchaomaster1 ceph]# kubectl apply -f cephfs-pod-1.yaml
创建第二个 pod，挂载 cephfs-pvc
[root@xianchaomaster1 ceph]# cat cephfs-pod-2.yaml 
apiVersion: v1
kind: Pod
metadata:name: cephfs-pod-2
spec:containers:- image: nginxname: nginximagePullPolicy: IfNotPresentvolumeMounts:- name: test-v1mountPath: /mntvolumes:- name: test-v1persistentVolumeClaim:claimName: cephfs-pvc