Merge pull request #163 from FlowFuse/persisten-storage

Persistent storage
FlowFuse · Jul 2, 2024 · 8ccd098 · 8ccd098
2 parents a931157 + 895ee75
commit 8ccd098
Show file tree

Hide file tree

Showing 6 changed files with 2,495 additions and 170 deletions.
diff --git a/README.md b/README.md
@@ -26,6 +26,10 @@ driver:
       cnameTarget: custom-loadbalancer.example.com
       certManagerIssuer: lets-encrypt
       ingressClass: custom-nginx
+    storage:
+      enabled: true
+      storageClass: nfs-storage
+      size: 5Gi
 ```
 
 - `registry` is the Docker Registry to load Stack Containers from
@@ -44,6 +48,10 @@ AWS EKS specific annotation for ALB Ingress. or `openshift` to allow running on
 - `customHostname.cnameTarget` The hostname users should configure their DNS entries to point at. Required. (default not set)
 - `customHostname.certManagerIssuer` Name of the Cluster issuer to use to create HTTPS certs for the custom hostname (default not set)
 - `customHostname.ingressClass` Name of the IngressClass to use to expose the custom hostname (default not set)
+- `storage.enabled` Mounts a persistent volume on `/data/storage` (default false)
+- `storage.storageClass` Name of StorageClass to use to allocate the volume (default not set)
+- `storage.storageClassEFSTag` Used instead of `storage.storageClass` when needing to shard across multiple EFS file systems (default not set)
+- `storage.size` Size of the volume to request (default not set)
 
 Expects to pick up K8s credentials from the environment
 

diff --git a/kubernetes.js b/kubernetes.js
@@ -1,6 +1,15 @@
 const got = require('got')
 const k8s = require('@kubernetes/client-node')
 const _ = require('lodash')
+const awsEFS = require('./lib/aws-efs.js')
+
+const {
+    deploymentTemplate,
+    serviceTemplate,
+    ingressTemplate,
+    customIngressTemplate,
+    persistentVolumeClaimTemplate
+} = require('./templates.js')
 
 /**
  * Kubernates Container driver
@@ -14,153 +23,6 @@ const _ = require('lodash')
  *
  */
 
-const deploymentTemplate = {
-    apiVersion: 'apps/v1',
-    kind: 'Deployment',
-    metadata: {
-        // name: "k8s-client-test-deployment",
-        labels: {
-            // name: "k8s-client-test-deployment",
-            nodered: 'true'
-            // app: "k8s-client-test-deployment"
-        }
-    },
-    spec: {
-        replicas: 1,
-        selector: {
-            matchLabels: {
-                // app: "k8s-client-test-deployment"
-            }
-        },
-        template: {
-            metadata: {
-                labels: {
-                    // name: "k8s-client-test-deployment",
-                    nodered: 'true'
-                    // app: "k8s-client-test-deployment"
-                }
-            },
-            spec: {
-                securityContext: {
-                    runAsUser: 1000,
-                    runAsGroup: 1000,
-                    fsGroup: 1000
-                },
-                containers: [
-                    {
-                        resources: {
-                            requests: {
-                                // 10th of a core
-                                cpu: '100m',
-                                memory: '128Mi'
-                            },
-                            limits: {
-                                cpu: '125m',
-                                memory: '192Mi'
-                            }
-                        },
-                        name: 'node-red',
-                        // image: "docker-pi.local:5000/bronze-node-red",
-                        imagePullPolicy: 'Always',
-                        env: [
-                            // {name: "APP_NAME", value: "test"},
-                            { name: 'TZ', value: 'Europe/London' }
-                        ],
-                        ports: [
-                            { name: 'web', containerPort: 1880, protocol: 'TCP' },
-                            { name: 'management', containerPort: 2880, protocol: 'TCP' }
-                        ],
-                        securityContext: {
-                            allowPrivilegeEscalation: false
-                        }
-                    }
-                ]
-            },
-            enableServiceLinks: false
-        }
-    }
-}
-
-const serviceTemplate = {
-    apiVersion: 'v1',
-    kind: 'Service',
-    metadata: {
-        // name: "k8s-client-test-service"
-    },
-    spec: {
-        type: 'ClusterIP',
-        selector: {
-            // name: "k8s-client-test"
-        },
-        ports: [
-            { name: 'web', port: 1880, protocol: 'TCP' },
-            { name: 'management', port: 2880, protocol: 'TCP' }
-        ]
-    }
-}
-
-const ingressTemplate = {
-    apiVersion: 'networking.k8s.io/v1',
-    kind: 'Ingress',
-    metadata: {
-        // name: "k8s-client-test-ingress",
-        // namespace: 'flowforge',
-        annotations: process.env.INGRESS_ANNOTATIONS ? JSON.parse(process.env.INGRESS_ANNOTATIONS) : {}
-    },
-    spec: {
-        ingressClassName: process.env.INGRESS_CLASS_NAME ? process.env.INGRESS_CLASS_NAME : null,
-        rules: [
-            {
-                // host: "k8s-client-test" + "." + "ubuntu.local",
-                http: {
-                    paths: [
-                        {
-                            pathType: 'Prefix',
-                            path: '/',
-                            backend: {
-                                service: {
-                                    // name: 'k8s-client-test-service',
-                                    port: { number: 1880 }
-                                }
-                            }
-                        }
-                    ]
-                }
-            }
-        ]
-    }
-}
-
-const customIngressTemplate = {
-    apiVersion: 'networking.k8s.io/v1',
-    kind: 'Ingress',
-    metadata: {
-        annotations: {}
-    },
-    spec: {
-        rules: [
-            {
-                http: {
-                    paths: [
-                        {
-                            pathType: 'Prefix',
-                            path: '/',
-                            backend: {
-                                service: {
-                                    port: { number: 1880 }
-                                }
-                            }
-                        }
-                    ]
-                }
-            }
-        ],
-        tls: [
-
-        ]
-    }
-}
-
 const createDeployment = async (project, options) => {
     const stack = project.ProjectStack.properties
 
@@ -276,7 +138,7 @@ const createDeployment = async (project, options) => {
         })
     }
 
-    if (this._app.config.driver.options.privateCA) {
+    if (this._app.config.driver.options?.privateCA) {
         localPod.spec.containers[0].volumeMounts = [
             {
                 name: 'cacert',
@@ -295,6 +157,29 @@ const createDeployment = async (project, options) => {
         localPod.spec.containers[0].env.push({ name: 'NODE_EXTRA_CA_CERTS', value: '/usr/local/ssl-certs/chain.pem' })
     }
 
+    if (this._app.config.driver.options?.storage?.enabled) {
+        const volMount = {
+            name: 'persistence',
+            mountPath: '/data/storage'
+        }
+        const vol = {
+            name: 'persistence',
+            persistentVolumeClaim: {
+                claimName: `${project.id}-pvc`
+            }
+        }
+        if (Array.isArray(localPod.spec.containers[0].volumeMounts)) {
+            localPod.spec.containers[0].volumeMounts.push(volMount)
+        } else {
+            localPod.spec.containers[0].volumeMounts = [volMount]
+        }
+        if (Array.isArray(localPod.spec.volumes)) {
+            localPod.spec.volumes.push(vol)
+        } else {
+            localPod.spec.volumes = [vol]
+        }
+    }
+
     if (this._app.license.active() && this._cloudProvider === 'openshift') {
         localPod.spec.securityContext = {}
     }
@@ -414,13 +299,57 @@ const createCustomIngress = async (project, hostname, options) => {
     return customIngress
 }
 
+const createPersistentVolumeClaim = async (project, options) => {
+    const namespace = this._app.config.driver.options?.projectNamespace || 'flowforge'
+    const pvc = JSON.parse(JSON.stringify(persistentVolumeClaimTemplate))
+
+    const drvOptions = this._app.config.driver.options
+
+    if (drvOptions?.storage?.storageClass) {
+        pvc.spec.storageClassName = drvOptions.storage.storageClass
+    } else if (drvOptions?.storage?.storageClassEFSTag) {
+        pvc.spec.storageClassName = await awsEFS.lookupStorageClass(drvOptions?.storage?.storageClassEFSTag)
+    }
+
+    if (drvOptions?.storage?.size) {
+        pvc.spec.resources.requests.storage = drvOptions.storage.size
+    }
+
+    pvc.metadata.namespace = namespace
+    pvc.metadata.name = `${project.id}-pvc`
+    pvc.metadata.labels = {
+        'ff-project-id': project.id,
+        'ff-project-name': project.safeName
+    }
+    console.log(`PVC: ${JSON.stringify(pvc, null, 2)}`)
+    return pvc
+}
+
 const createProject = async (project, options) => {
     const namespace = this._app.config.driver.options.projectNamespace || 'flowforge'
 
     const localDeployment = await createDeployment(project, options)
     const localService = await createService(project, options)
     const localIngress = await createIngress(project, options)
 
+    if (this._app.config.driver.options?.storage?.enabled) {
+        const localPVC = await createPersistentVolumeClaim(project, options)
+        // console.log(JSON.stringify(localPVC, null, 2))
+        try {
+            await this._k8sApi.createNamespacedPersistentVolumeClaim(namespace, localPVC)
+        } catch (err) {
+            if (err.statusCode === 409) {
+                this._app.log.warn(`[k8s] PVC for instance ${project.id} already exists, proceeding...`)
+            } else {
+                if (project.state !== 'suspended') {
+                    this._app.log.error(`[k8s] Instance ${project.id} - error creating PVC: ${err.toString()} ${err.statusCode}`)
+                    // console.log(err)
+                    throw err
+                }
+            }
+        }
+    }
+
     try {
         await this._k8sAppApi.createNamespacedDeployment(namespace, localDeployment)
     } catch (err) {
@@ -838,6 +767,15 @@ module.exports = {
             await this._k8sApi.deleteNamespacedPod(project.safeName, this._namespace)
         }
 
+        // We should not delete the PVC when the instance is suspended
+        // if (this._app.config.driver.options?.storage?.enabled) {
+        //     try {
+        //         await this._k8sApi.deleteNamespacedPersistentVolumeClaim(`${project.safeName}-pvc`, this._namespace)
+        //     } catch (err) {
+        //         this._app.log.error(`[k8s] Instance ${project.id} - error deleting PVC: ${err.toString()} ${err.statusCode}`)
+        //     }
+        // }
+
         this._projects[project.id].state = 'suspended'
         return new Promise((resolve, reject) => {
             let counter = 0
@@ -921,6 +859,14 @@ module.exports = {
                 }
             }
         }
+        if (this._app.config.driver.options?.storage?.enabled) {
+            try {
+                await this._k8sApi.deleteNamespacedPersistentVolumeClaim(`${project.id}-pvc`, this._namespace)
+            } catch (err) {
+                this._app.log.error(`[k8s] Instance ${project.id} - error deleting PVC: ${err.toString()} ${err.statusCode}`)
+                // console.log(err)
+            }
+        }
         delete this._projects[project.id]
     },
     /**

diff --git a/lib/aws-efs.js b/lib/aws-efs.js
@@ -0,0 +1,54 @@
+const { EFSClient, DescribeFileSystemsCommand, DescribeAccessPointsCommand } = require("@aws-sdk/client-efs")
+
+let client
+
+async function lookupStorageClass (tagName) {
+
+    // console.log(`Looking for ${tagName}`)
+
+    if (!client) {
+        client = new EFSClient()
+    }
+
+    const fsCommand = new DescribeFileSystemsCommand()
+    const fsList = await client.send(fsCommand)
+    // console.log(JSON.stringify(fsList, null, 2))
+
+    const fileSystems = []
+
+    for (let i = 0; i<fsList.FileSystems.length; i++) {
+        let found = false
+        let storageClass = ''
+        for (let j = 0; j<fsList.FileSystems[i].Tags.length; j++) {
+            const tag = fsList.FileSystems[i].Tags[j]
+            if (tag.Key === tagName) {
+                found = true
+            } 
+            if (tag.Key === 'storage-class-name') {
+                storageClass = tag.Value
+            }
+        }
+        if (found) {
+            // console.log(storageClass)
+            const apParams = {
+                FileSystemId: fsList.FileSystems[i].FileSystemId
+            }
+            // console.log(apParams)
+            const apListCommand = new DescribeAccessPointsCommand(apParams)
+            const apList = await client.send(apListCommand)
+            // fileSystems[fsList.FileSystems[i].FileSystemId] 
+            fileSystems.push({
+                apCount: apList.AccessPoints.length,
+                storageClass
+            })
+        } 
+    }
+    fileSystems.sort((a,b) => a.apCount - b.apCount)
+
+    return fileSystems[0]?.storageClass
+}
+
+
+module.exports = {
+    lookupStorageClass
+}