diff --git a/deploy/helm/seaweedfs-csi-driver/.helmignore b/deploy/helm/seaweedfs-csi-driver/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/helm/seaweedfs-csi-driver/Chart.yaml b/deploy/helm/seaweedfs-csi-driver/Chart.yaml new file mode 100644 index 0000000..2a99ae9 --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: seaweedfs-csi-driver +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +appVersion: latest diff --git a/deploy/helm/seaweedfs-csi-driver/templates/_helpers.tpl b/deploy/helm/seaweedfs-csi-driver/templates/_helpers.tpl new file mode 100644 index 0000000..642cee9 --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/_helpers.tpl @@ -0,0 +1,3 @@ +{{- define "seaweedfs-csi-driver.name" -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/deploy/helm/seaweedfs-csi-driver/templates/configmap-security.yaml b/deploy/helm/seaweedfs-csi-driver/templates/configmap-security.yaml new file mode 100644 index 0000000..ecdcf8b --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/configmap-security.yaml @@ -0,0 +1,61 @@ +{{- if .Values.tlsSecret }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }} + labels: + app: {{ template "seaweedfs-csi-driver.name" . }} +data: + security.toml: |- + # this file is read by master, volume server, and filer + + # the jwt signing key is read by master and volume server + # a jwt expires in 10 seconds + #[jwt.signing] + # key = "{{ .Values.jwtSigningKey }}" + #expires_after_seconds = 10 # seconds + + #[jwt.signing.read] + #key = "" + #expires_after_seconds = 10 # seconds + # all grpc tls authentications are mutual + # the values for the following ca, cert, and key are paths to the PERM files. + [grpc] + ca = "/usr/local/share/ca-certificates/ca.crt" + + [grpc.volume] + cert = "/usr/local/share/ca-certificates/tls.crt" + key = "/usr/local/share/ca-certificates/tls.key" + ca = "/usr/local/share/ca-certificates/ca.crt" + + [grpc.master] + cert = "/usr/local/share/ca-certificates/tls.crt" + key = "/usr/local/share/ca-certificates/tls.key" + ca = "/usr/local/share/ca-certificates/ca.crt" + + [grpc.filer] + cert = "/usr/local/share/ca-certificates/tls.crt" + key = "/usr/local/share/ca-certificates/tls.key" + ca = "/usr/local/share/ca-certificates/ca.crt" + + [grpc.msg_broker] + cert = "/usr/local/share/ca-certificates/tls.crt" + key = "/usr/local/share/ca-certificates/tls.key" + ca = "/usr/local/share/ca-certificates/ca.crt" + + # use this for any place needs a grpc client + # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" + [grpc.client] + cert = "/usr/local/share/ca-certificates/tls.crt" + key = "/usr/local/share/ca-certificates/tls.key" + ca = "/usr/local/share/ca-certificates/ca.crt" + + # volume server https options + # Note: work in progress! + # this does not work with other clients, e.g., "weed filer|mount" etc, yet. + #[https.client] + #enabled = false + #[https.volume] + #cert = "" + #key = "" +{{- end }} diff --git a/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yml b/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yml new file mode 100644 index 0000000..561cb2b --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.driverName }} +spec: + attachRequired: true + podInfoOnMount: true diff --git a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml new file mode 100644 index 0000000..aa887c2 --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml @@ -0,0 +1,112 @@ +{{- if .Values.node.enabled}} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-node +spec: + selector: + matchLabels: + app: {{ template "seaweedfs-csi-driver.name" . }}-node + updateStrategy: + rollingUpdate: + maxUnavailable: 25% + template: + metadata: + labels: + app: {{ template "seaweedfs-csi-driver.name" . }}-node + spec: + priorityClassName: system-node-critical + serviceAccountName: {{ template "seaweedfs-csi-driver.name" . }}-node-sa + #hostNetwork: true + #dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: {{ .Values.csiNodeDriverRegistrar.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/{{ .Values.driverName }}/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + {{ toYaml .Values.csiNodeDriverRegistrar.resources | nindent 12 }} + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: csi-seaweedfs-plugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{.Values.seaweedfsCsiPlugin.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--filer=$(SEAWEEDFS_FILER)" + - "--nodeid=$(NODE_ID)" + - "-v=9" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: SEAWEEDFS_FILER + value: {{ .Values.seaweedfsFiller | quote }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.tlsSecret }} + - name: WEED_GRPC_CLIENT_KEY + value: /var/run/secrets/app/tls/tls.key + - name: WEED_GRPC_CLIENT_CERT + value: /var/run/secrets/app/tls/tls.crt + - name: WEED_GRPC_CA + value: /var/run/secrets/app/tls/ca.crt + {{- end }} + resources: + {{ toYaml .Values.seaweedfsCsiPlugin.resources | nindent 12 }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /dev + name: device-dir + {{- if .Values.tlsSecret }} + - name: tls + mountPath: /var/run/secrets/app/tls + {{- end }} + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/{{ .Values.driverName }} + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: device-dir + hostPath: + path: /dev + {{- if .Values.tlsSecret }} + - name: tls + secret: + secretName: {{ .Values.tlsSecret }} + {{- end }} +{{- end }} diff --git a/deploy/helm/seaweedfs-csi-driver/templates/rbac.yml b/deploy/helm/seaweedfs-csi-driver/templates/rbac.yml new file mode 100644 index 0000000..fced728 --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/rbac.yml @@ -0,0 +1,172 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-provisioner-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-provisioner-binding +subjects: + - kind: ServiceAccount + name: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "seaweedfs-csi-driver.name" . }}-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments", "volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-attacher-binding +subjects: + - kind: ServiceAccount + name: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "seaweedfs-csi-driver.name" . }}-attacher-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-snapshotter-binding +subjects: + - kind: ServiceAccount + name: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "seaweedfs-csi-driver.name" . }}-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-controller-role +rules: + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-controller-binding +subjects: + - kind: ServiceAccount + name: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-controller-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-node-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-node-binding +subjects: + - kind: ServiceAccount + name: {{ template "seaweedfs-csi-driver.name" . }}-node-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/helm/seaweedfs-csi-driver/templates/serviceaccounts.yml b/deploy/helm/seaweedfs-csi-driver/templates/serviceaccounts.yml new file mode 100644 index 0000000..6576c4e --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/serviceaccounts.yml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-node-sa diff --git a/deploy/helm/seaweedfs-csi-driver/templates/statefulset.yml b/deploy/helm/seaweedfs-csi-driver/templates/statefulset.yml new file mode 100644 index 0000000..afdef7d --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/statefulset.yml @@ -0,0 +1,93 @@ +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: {{ template "seaweedfs-csi-driver.name" . }}-controller +spec: + selector: + matchLabels: + app: {{ template "seaweedfs-csi-driver.name" . }}-controller + serviceName: "csi-seaweedfs" + replicas: 1 + template: + metadata: + labels: + app: {{ template "seaweedfs-csi-driver.name" . }}-controller + spec: + priorityClassName: system-cluster-critical + serviceAccountName: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa + containers: + # provisioner + - name: csi-provisioner + image: {{ .Values.csiProvisioner.image }} + args: + - "--csi-address=$(ADDRESS)" + - -v + - "9" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: {{ .Values.imagePullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{ toYaml .Values.csiProvisioner.resources | nindent 12 }} + # attacher + - name: csi-attacher + image: {{ .Values.csiAttacher.image }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--timeout=120s" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: {{ .Values.imagePullPolicy }} + resources: + {{ toYaml .Values.csiAttacher.resources | nindent 12 }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + # SeaweedFs Plugin + - name: seaweedfs-csi-plugin + image: {{.Values.seaweedfsCsiPlugin.image}} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--filer=$(SEAWEEDFS_FILER)" + - "--nodeid=$(NODE_ID)" + - -v + - "9" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: SEAWEEDFS_FILER + value: {{ .Values.seaweedfsFiller | quote }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.tlsSecret }} + - name: WEED_GRPC_CLIENT_KEY + value: /var/run/secrets/app/tls/tls.key + - name: WEED_GRPC_CLIENT_CERT + value: /var/run/secrets/app/tls/tls.crt + - name: WEED_GRPC_CA + value: /var/run/secrets/app/tls/ca.crt + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if .Values.tlsSecret }} + - name: tls + mountPath: /var/run/secrets/app/tls + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} + {{- if .Values.tlsSecret }} + - name: tls + secret: + secretName: {{ .Values.tlsSecret }} + {{- end }} diff --git a/deploy/helm/seaweedfs-csi-driver/templates/storageclass.yml b/deploy/helm/seaweedfs-csi-driver/templates/storageclass.yml new file mode 100644 index 0000000..c79c29d --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/storageclass.yml @@ -0,0 +1,10 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .Values.storageClassName }} + namespace: {{ .Release.Namespace }} + annotations: + {{- if .Values.isDefaultStorageClass }} + storageclass.kubernetes.io/is-default-class: "true" + {{- end }} +provisioner: {{ .Values.driverName }} diff --git a/deploy/helm/seaweedfs-csi-driver/values.yaml b/deploy/helm/seaweedfs-csi-driver/values.yaml new file mode 100644 index 0000000..f594151 --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/values.yaml @@ -0,0 +1,32 @@ + +# host and port of your SeaweedFs filer +seaweedfsFiller: "" +storageClassName: seaweedfs-storage +isDefaultStorageClass: false +tlsSecret: "" + +imagePullPolicy: "IfNotPresent" + +csiProvisioner: + image: quay.io/k8scsi/csi-provisioner:v1.6.1 + resources: {} + +csiAttacher: + image: quay.io/k8scsi/csi-attacher:v1.2.1 + resources: {} + +csiNodeDriverRegistrar: + image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + resources: {} + +seaweedfsCsiPlugin: + image: chrislusf/seaweedfs-csi-driver:latest + resources: {} + +# NOT Change, for future releases. Must be equal Name in GetPluginInfoResponse +driverName: seaweedfs-csi-driver + +node: + # Deploy node daemonset + # for a smoother update it is better to use static pods on nodes + enabled: true