Self heal NGINX and PHP

package.json

@@ -1,6 +1,6 @@
 {
   "name": "domcloud-bridge",
-  "version": "0.64.0",
+  "version": "0.65.0",
   "description": "Deployment runner for DOM Cloud",
   "main": "app.js",
   "engines": {

src/controllers/status.js

@@ -7,6 +7,7 @@ import {
 } from '../util.js';
 import express from 'express';
 import path from 'path';
+import { fixNGINX, fixPHP } from '../executor/pulse.js';
 
 const tmpCheck = path.join(process.cwd(), '/.tmp/check')
 const tmpTest = path.join(process.cwd(), '/.tmp/test')
@@ -20,7 +21,6 @@ let lastTest = 0;
 let lastTestResult = {};
 let lastTestOK = false;
 
-
 export default function () {
     var router = express.Router();
     router.get('/about', async function (req, res, next) {
@@ -39,6 +39,28 @@ export default function () {
                 lastCheck = Date.now();
             }
             res.status(lastCheckOK ? 200 : 500).json(lastCheckResult);
+            if (lastCheckOK) {
+                return;
+            }
+            let lastTestResult = null;
+            for (const [key, val] of Object.entries(lastCheckResult.statuses)) {
+                if (val != "failed") {
+                    continue;
+                }
+                if (key.endsWith("-php-fpm")) {
+                    if (!lastTestResult) {
+                        await spawnSudoUtil('SHELL_TEST');
+                        lastTestResult = JSON.parse(cat(tmpTest));
+                    }
+                    await fixPHP(lastTestResult);
+                } else if (key == "nginx") {
+                    if (!lastTestResult) {
+                        await spawnSudoUtil('SHELL_TEST');
+                        lastTestResult = JSON.parse(cat(tmpTest));
+                    }
+                    await fixNGINX(lastTestResult);
+                }
+            }
         } catch (error) {
             next(error);
         }

src/executor/pulse.js

@@ -0,0 +1,37 @@
+import { spawnSudoUtil } from "../util.js";
+
+export async function fixPHP(test) {
+    for (const [name, logs] of Object.entries(test.logs.fpms)) {
+        const [lastLog] = logs.splice(logs.length - 1);
+        if (lastLog.endsWith("ERROR: FPM initialization failed")) {
+            for (const element of logs) {
+                let m = element.match(/\[pool (\d+)\] cannot get uid for user '(.+)'/);
+                if (m) {
+                    await spawnSudoUtil("CLEAN_DOMAIN", ["mv", m[1], ""]);
+                }
+            }
+        }
+    }
+}
+
+export async function fixNGINX(test) {
+    const logs = test.logs.nginx;
+    const [lastLog] = logs.splice(logs.length - 1);
+    if (lastLog == "nginx: configuration file /etc/nginx/nginx.conf test failed") {
+        for (const element of logs) {
+            let m = element.match(/nginx: \[emerg\] cannot load certificate \"([\w.\/]+)\"/);
+            if (m) {
+                let find = (await spawnSudoUtil("SHELL_SUDO", [
+                    "root", "grep", "-lr", m[1], "/etc/nginx/conf.d"
+                ])).stdout.trim().split("\n");
+                for (const f of find) {
+                    let m2 = f.match(/\/etc\/nginx\/conf.d\/(.+)\.conf/);
+                    if (m2) {
+                        await spawnSudoUtil("CLEAN_DOMAIN", ["mv", "", m2[1]]);
+                    }
+                }
+            }
+        }
+    }
+}
+

src/executor/runner.js

@@ -273,7 +273,7 @@ export default async function runConfig(config, domain, writer, sandbox = false)
                         await virtExec("delete-domain", value, {
                             domain,
                         });
-                        await spawnSudoUtil('CLEAN_DOMAIN', [domaindata['ID'], domain]);
+                        await spawnSudoUtil('CLEAN_DOMAIN', ["rm", domaindata['ID'], domain]);
                         // no need to do other stuff
                         return;
                     default:

sudoutil.js

@@ -226,20 +226,37 @@ switch (cli.args.shift()) {
         ShellString(cnf).to(env.OPENSSL_OUT);
         exit(0);
     case 'CLEAN_DOMAIN':
+        if (cli.args.length < 3) {
+            console.error("CLEAN_DOMAIN [rm|mv] [id] [domain]");
+            exit(1);
+        }
+        const mode = cli.args.shift();
         const id = cli.args.shift();
         const domain = cli.args.shift();
         var fpmlist = ls(env.PHPFPM_REMILIST).filter((f) => f.match(/php\d\d/));
         var fpmcleaned = '', nginxcleaned = '';
-        for (const f of fpmlist) {
-            var p = `${env.PHPFPM_REMICONF.replace('$', f)}/${id}.conf`;
-            if (existsSync(p)) {
+        /**
+         * @param {string} p
+         */
+        function cleanfile(p) {
+            if (mode == "rm") {
                 rm(p);
-                fpmcleaned = f;
-                break;
+            } else {
+                mv(p, p + ".bak");
+            }
+        }
+        if (id) {
+            for (const f of fpmlist) {
+                var p = `${env.PHPFPM_REMICONF.replace('$', f)}/${id}.conf`;
+                if (existsSync(p)) {
+                    cleanfile(p);
+                    fpmcleaned = f;
+                    break;
+                }
             }
         }
-        if (existsSync(env.NGINX_PATH.replace('$', domain))) {
-            rm(p);
+        if (domain && existsSync(env.NGINX_PATH.replace('$', domain))) {
+            cleanfile(p);
             nginxcleaned = '1';
         }
         if (fpmcleaned) {
@@ -371,14 +388,14 @@ switch (cli.args.shift()) {
                 quota: quotaOK ? 0 : 1,
             },
             logs: {
-                cpuinfo: chkcpu.stdout.trim().split('\n'),
+                cpuinfo: chkcpu.stdout.trimEnd().split('\n'),
                 meminfo: chkmem.stdout.trimEnd().split('\n'),
-                storage: storage.stdout.trim().split('\n'),
+                storage: storage.stdout.trimEnd().split('\n'),
                 quota,
-                nginx: nginx.stderr.trim().split('\n'),
-                fpms: fpms.map((f) => f.stderr.trim()),
-                iptables: iptables.stderr.trim().split('\n'),
-                ip6tables: ip6tables.stderr.trim().split('\n'),
+                nginx: nginx.stderr.trimEnd().split('\n'),
+                fpms: Object.fromEntries(fpmlist.map((_, i) => [fpmlist[i], fpms[i].stderr.trimEnd().split('\n')])),
+                iptables: iptables.stderr.trimEnd().split('\n'),
+                ip6tables: ip6tables.stderr.trimEnd().split('\n'),
             },
         })).to(env.SHELLTEST_TMP);
         exit(0);