Linstor это система управления блочными устройствами (дисками). Данная система позволяет организовывать распределенное хранилище данных (DRBD).
Если мы устанавливаем в talos- необходимо использовать образ с включенным DRBD (через image factory). Дополнительно в патче воркера дописать DRBD модуль:
machine:
type: worker
kernel:
modules:
- name: drbd
parameters:
- usermode_helper=disabled
- name: drbd_transport_tcp
network:
hostname: w-01
После для деплоя необходимо установить в кластер оператора:
kubectl apply --server-side -f "https://github.com/piraeusdatastore/piraeus-operator/releases/latest/download/manifest.yaml"
Когда мы применим оператора- создадутся поды и деплойменты в ns piraeus-datastore
Далее необходимо задеплоить сателлиты, cert-manager (для передачи данных по tls), storageClass, и кластер (где мы запретим размещение элементов на мастер нодах).
Подробнее тут: https://piraeus.io/docs/stable/
Делаем это следующими файлами деплоя:
kustomization.yaml:
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- cert-manager.yaml
- cluster.yaml
- satelite.yaml
- sc.yaml
namespace.yaml:
apiVersion: v1
kind: Namespace
metadata:
name: piraeus-datastore
labels:
pod-security.kubernetes.io/enforce: privileged
cert-namager.yaml
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: ca-bootstrapper
namespace: piraeus-datastore
spec:
selfSigned: { }
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: linstor-internal-ca
namespace: piraeus-datastore
spec:
commonName: linstor-internal-ca
secretName: linstor-internal-ca
duration: 87600h
renewBefore: 87550h
isCA: true
usages:
- signing
- key encipherment
- cert sign
issuerRef:
name: ca-bootstrapper
kind: Issuer
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: linstor-internal-ca
namespace: piraeus-datastore
spec:
ca:
secretName: linstor-internal-ca
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: linstor-api-ca
namespace: piraeus-datastore
spec:
commonName: linstor-api-ca
secretName: linstor-api-ca
duration: 87600h
renewBefore: 87550h
isCA: true
usages:
- signing
- key encipherment
- cert sign
issuerRef:
name: ca-bootstrapper
kind: Issuer
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: linstor-api-ca
namespace: piraeus-datastore
spec:
ca:
secretName: linstor-api-ca
cluster.yaml:
apiVersion: piraeus.io/v1
kind: LinstorCluster
metadata:
name: linstorcluster
spec:
properties:
- name: DrbdOptions/Net/tls
value: "yes"
- name: DrbdOptions/auto-quorum
value: "suspend-io"
- name: DrbdOptions/Resource/on-no-data-accessible
value: "suspend-io"
- name: DrbdOptions/Resource/on-suspended-primary-outdated
value: "force-secondary"
- name: DrbdOptions/Net/rr-conflict
value: "retry-connect"
- name: DrbdOptions/PeerDevice/c-max-rate
value: "500000"
- name: DrbdOptions/PeerDevice/c-min-rate
value: "100000"
- name: DrbdOptions/PeerDevice/resync-rate
value: "250000"
apiTLS:
certManager:
name: linstor-api-ca
kind: Issuer
internalTLS:
certManager:
name: linstor-internal-ca
kind: Issuer
csiController:
enabled: true
controller:
enabled: true
patches:
- target:
kind: Deployment
name: linstor-controller
patch: |
apiVersion: apps/v1
kind: Deployment
metadata:
name: linstor-controller
spec:
template:
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: "node-role.kubernetes.io/control-plane"
operator: "DoesNotExist"
- target:
kind: Deployment
name: linstor-csi-controller
patch: |
apiVersion: apps/v1
kind: Deployment
metadata:
name: linstor-csi-controller
spec:
template:
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: "node-role.kubernetes.io/control-plane"
operator: "DoesNotExist"
- target:
kind: DaemonSet
name: ha-controller
patch: |
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: ha-controller
spec:
template:
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: "node-role.kubernetes.io/control-plane"
operator: "DoesNotExist"
- target:
kind: DaemonSet
name: linstor-csi-node
patch: |
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: linstor-csi-node
spec:
template:
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: "node-role.kubernetes.io/control-plane"
operator: "DoesNotExist"
satelite.yaml:
---
apiVersion: piraeus.io/v1
kind: LinstorSatelliteConfiguration
metadata:
name: satellite
spec:
internalTLS:
tlsHandshakeDaemon: true
certManager:
name: linstor-internal-ca
kind: Issuer
podTemplate:
spec:
hostNetwork: true
tolerations:
- key: storage
operator: Equal
value: 'true'
effect: NoSchedule
initContainers:
- name: drbd-shutdown-guard
$patch: delete
- name: drbd-module-loader
$patch: delete
volumes:
- name: run-systemd-system
$patch: delete
- name: run-drbd-shutdown-guard
$patch: delete
- name: systemd-bus-socket
$patch: delete
- name: lib-modules
$patch: delete
- name: usr-src
$patch: delete
- name: etc-lvm-backup
hostPath:
path: /var/etc/lvm/backup
type: DirectoryOrCreate
- name: etc-lvm-archive
hostPath:
path: /var/etc/lvm/archive
type: DirectoryOrCreate
sc.yaml:
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ssd-pool
annotations:
storageclass.kubernetes.io/is-default-class: "true"
parameters:
property.linstor.csi.linbit.com/DrbdOptions/Net/tls: "yes"
csi.storage.k8s.io/fstype: xfs
linstor.csi.linbit.com/autoPlace: "1"
linstor.csi.linbit.com/storagePool: ssd_pool
linstor.csi.linbit.com/usePvcName: "true"
linstor.csi.linbit.com/mountOpts: discard
provisioner: linstor.csi.linbit.com
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true
После применения файлов или kubectl apply -k ./ дожидаемся деплоя в ns piraeus-datastore всех элементов.
После чего (для удобства) скачиваем утилиту:
wget https://github.com/piraeusdatastore/kubectl-linstor/releases/download/v0.3.2/kubectl-linstor_v0.3.2_linux_amd64.tar.gz && tar -xvf kubectl-linstor_v0.3.2_linux_amd64.tar.gz && rm LICENSE README.md && chmod 755 kubectl-linstor
Вообще не обязательно ее скачивать, но я буду использовать ее. Если нет желания- можно отправлять команды напрямую в linstor-controller, к примеру так: exec -it deployment/linstor-controller -n piraeus-datastore -- linstor resource list
Я же в свою очередь дальше буду юзать утилиту. Создаем на каждой ноде диск:
kubectl-linstor physical-storage create-device-pool --pool-name ssd_pool LVM w-01 /dev/sdb --storage-pool ssd_pool (системный диск юзать нельзя)
И проверяем статус:
root@prod1:/opt/talos# kubectl-linstor storage-pool list
E0922 22:22:09.243070 460832 websocket.go:297] Unknown stream id 1, discarding message
╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
┊ StoragePool ┊ Node ┊ Driver ┊ PoolName ┊ FreeCapacity ┊ TotalCapacity ┊ CanSnapshots ┊ State ┊ SharedName ┊
╞═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡
┊ DfltDisklessStorPool ┊ w-01 ┊ DISKLESS ┊ ┊ ┊ ┊ False ┊ Ok ┊ w-01;DfltDisklessStorPool ┊
┊ DfltDisklessStorPool ┊ w-02 ┊ DISKLESS ┊ ┊ ┊ ┊ False ┊ Ok ┊ w-02;DfltDisklessStorPool ┊
┊ DfltDisklessStorPool ┊ w-03 ┊ DISKLESS ┊ ┊ ┊ ┊ False ┊ Ok ┊ w-03;DfltDisklessStorPool ┊
┊ ssd_pool ┊ w-01 ┊ LVM ┊ ssd_pool ┊ 160.00 GiB ┊ 160.00 GiB ┊ False ┊ Ok ┊ w-01;ssd_pool ┊
┊ ssd_pool ┊ w-02 ┊ LVM ┊ ssd_pool ┊ 158.99 GiB ┊ 160.00 GiB ┊ False ┊ Ok ┊ w-02;ssd_pool ┊
┊ ssd_pool ┊ w-03 ┊ LVM ┊ ssd_pool ┊ 160.00 GiB ┊ 160.00 GiB ┊ False ┊ Ok ┊ w-03;ssd_pool ┊
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
После этого можно пользоваться)
Команды чтобы не забыть:
kubectl-linstor physical-storage create-device-pool --pool-name ssd_pool LVM w-01 /dev/sda --storage-pool ssd_pool
kubectl exec -it deployment/linstor-controller -n piraeus-datastore -- linstor volume list
kubectl-linstor node-connection drbd-peer-options w-02 w-03 --tls yes
kubectl-linstor volume list
kubectl-linstor node list
kubectl-linstor resource list
kubectl-linstor storage-pool list
kubectl-linstor physical-storage list
kubectl-linstor error-reports list --nodes twrm-01 --since 1h
kubectl-linstor resource-definition create test-volume
kubectl-linstor volume-definition create test-volume --storage-pool ssd-pool-sdc 1G
kubectl-linstor resource create test-volume twrm-01 --storage-pool ssd-pool-sdc