WIP recursive queries

src/ORM/Connect/DBQueryBuilder.php

@@ -74,6 +74,7 @@ protected function buildSelectQuery(SQLSelect $query, array &$parameters)
         if ($needsParenthisis) {
             $sql .= "({$nl}";
         }
+        $sql .= $this->buildWithFragment($query, $parameters);
         $sql .= $this->buildSelectFragment($query, $parameters);
         $sql .= $this->buildFromFragment($query, $parameters);
         $sql .= $this->buildWhereFragment($query, $parameters);
@@ -165,6 +166,38 @@ protected function buildUpdateQuery(SQLUpdate $query, array &$parameters)
         return $sql;
     }
 
+    /**
+     * Returns the WITH clauses ready for inserting into a query.
+     */
+    protected function buildWithFragment(SQLSelect $query, array &$parameters): string
+    {
+        $with = $query->getWith();
+        if (empty($with)) {
+            return '';
+        }
+
+        $nl = $this->getSeparator();
+        $clauses = [];
+
+        foreach ($with as $name => $bits) {
+            $clause = $bits['recursive'] ? 'RECURSIVE ' : '';
+            $clause .= $name;
+
+            if (!empty($bits['cte_fields'])) {
+                $clause .= ' (' . implode(', ', $bits['cte_fields']) . ')';
+            }
+
+            $clause .= " AS ({$nl}";
+
+            $clause .= $this->buildSelectQuery($bits['query'], $parameters);
+
+            $clause .= "{$nl})";
+            $clauses[] = $clause;
+        }
+
+        return 'WITH ' . implode(",{$nl}", $clauses) . $nl;
+    }
+
     /**
      * Returns the SELECT clauses ready for inserting into a query.
      *

src/ORM/Connect/Database.php

@@ -636,6 +636,17 @@ abstract public function searchEngine(
         $invertedMatch = false
     );
 
+    /**
+     * Determines if this database supports Common Table Expression (aka WITH) clauses.
+     * By default it is assumed that it doesn't unless this method is explicitly overridden.
+     *
+     * @param bool $recursive if true, checks specifically if recursive CTEs are supported.
+     */
+    public function supportsCteQueries(bool $recursive = false): bool
+    {
+        return false;
+    }
+
     /**
      * Determines if this database supports transactions
      *
@@ -654,7 +665,6 @@ public function supportsSavepoints()
         return false;
     }
 
-
     /**
      * Determines if the used database supports given transactionMode as an argument to startTransaction()
      * If transactions are completely unsupported, returns false.

src/ORM/Connect/MySQLDatabase.php

@@ -313,6 +313,41 @@ public function searchEngine(
         return $list;
     }
 
+    public function supportsCteQueries(bool $recursive = false): bool
+    {
+        $version = $this->getVersion();
+        $mariaDBVersion = $this->getMariaDBVersion($version);
+        if ($mariaDBVersion) {
+            // MariaDB has supported CTEs since 10.2.1, and recursive CTEs from 10.2.2
+            // see https://mariadb.com/kb/en/mariadb-1021-release-notes/ and https://mariadb.com/kb/en/mariadb-1022-release-notes/
+            $supportedFrom = $recursive ? '10.2.2' : '10.2.1';
+            return $this->compareVersion($mariaDBVersion, $supportedFrom) >= 0;
+        }
+        // MySQL has supported both kinds of CTEs since 8.0.1
+        // see https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-1.html
+        return $this->compareVersion($version, '8.0.1') >= 0;
+    }
+
+    private function getMariaDBVersion(string $version): ?string
+    {
+        // MariaDB versions look like "5.5.5-10.6.8-mariadb-1:10.6.8+maria~focal"
+        // or "10.8.3-MariaDB-1:10.8.3+maria~jammy"
+        // The relevant part is the x.y.z-mariadb portion.
+        if (!preg_match('/((\d+\.){2}\d+)-mariadb/i', $version, $matches)) {
+            return null;
+        }
+        return $matches[1];
+    }
+
+    private function compareVersion(string $actualVersion, string $atLeastVersion): int
+    {
+        // Assume it's lower if it's not a proper version number
+        if (!preg_match('/^(\d+\.){2}\d+$/', $actualVersion)) {
+            return -1;
+        }
+        return version_compare($actualVersion, $atLeastVersion);
+    }
+
     /**
      * Returns the TransactionManager to handle transactions for this database.
      *

src/ORM/DataQuery.php

@@ -701,8 +701,6 @@ public function disjunctiveGroup()
         return new DataQuery_SubGroup($this, 'OR', $clause);
     }
 
-
-
     /**
      * Create a conjunctive subgroup
      *
@@ -723,6 +721,61 @@ public function conjunctiveGroup()
         return new DataQuery_SubGroup($this, 'AND', $clause);
     }
 
+    /**
+     * Adds a Common Table Expression (CTE), aka WITH clause.
+     *
+     * Use of this method should usually be within a conditional check against DB::get_conn()->supportsCteQueries().
+     *
+     * @param string $name The name of the WITH clause, which can be referenced in any queries UNIONed to the $query
+     * and in this query directly, as though it were a table name.
+     * @param string[] $cteFields Aliases for any columns selected in $query which can be referenced in any queries
+     * UNIONed to the $query and in this query directly, as though they were columns in a real table.
+     * NOTE: If $query is a DataQuery, then cteFields must be the names of real columns on that DataQuery's data class.
+     * @param string|string[] $onClause The "ON" clause (escaped SQL statement) for an INNER JOIN on the query.
+     * It can either be a full clause (like you would pass to {@link leftJoin()} or {@link innerJoin()}),
+     * or it can be an array mapping of the field(s) on the dataclass table that map with the field(s) on the CTE table
+     * e.g. ['ID' => 'cte_id']
+     * If you want to use another join type, leave this blank and call the appropriate join method.
+     */
+    public function with(string $name, DataQuery|SQLSelect $query, array $cteFields = [], string|array $onClause = '', bool $recursive = false): static
+    {
+        $schema = DataObject::getSchema();
+
+        // If the query is a DataQuery, make sure all manipulators, joins, etc are applied
+        if ($query instanceof self) {
+            $cteDataClass = $query->dataClass();
+            $query = $query->query();
+            if (!empty($cteFields)) {
+                $selectFields = array_map(fn($colName) => $schema->sqlColumnForField($cteDataClass, $colName), $cteFields);
+                $query->setSelect($selectFields);
+            }
+        }
+
+        // Craft the "ON" clause for the join if we need to
+        if (is_array($onClause) && !empty($onClause)) {
+            $onClauses = [];
+            foreach ($onClause as $myField => $cteField) {
+                $onClauses[] = $schema->sqlColumnForField($this->dataClass(), $myField) . ' = ' . Convert::symbol2sql([$name, $cteField]);
+            }
+            $onClause = implode(' AND ', $onClauses);
+        }
+
+        // Ensure all cte fields are escaped correctly
+        array_walk($cteFields, function ($colName) {
+            return preg_match('/^".*"$/', $colName) ? $colName : Convert::symbol2sql($colName);
+        });
+
+        // Add the WITH clause
+        $this->query->addWith(Convert::symbol2sql($name), $query, $cteFields, $recursive);
+
+        // Only add a join if we have an ON clause, to allow developers to use their own alternative JOIN if they want to
+        if ($onClause) {
+            $this->query->addInnerJoin($name, $onClause);
+        }
+
+        return $this;
+    }
+
     /**
      * Adds a WHERE clause.
      *

src/ORM/Queries/SQLSelect.php

@@ -45,6 +45,18 @@ class SQLSelect extends SQLConditionalExpression
      */
     protected array $union = [];
 
+    /**
+     * An array of WITH clauses.
+     * This array is indexed with the name for the temporary table generated for the WITH clause,
+     * and contains data in the following format:
+     * [
+     *   'cte_fields' => string[],
+     *   'query' => SQLSelect|null,
+     *   'recursive' => boolean,
+     * ]
+     */
+    protected array $with = [];
+
     /**
      * If this is true DISTINCT will be added to the SQL.
      *
@@ -564,6 +576,37 @@ public function getUnions(): array
         return $this->union;
     }
 
+    /**
+     * Adds a Common Table Expression (CTE), aka WITH clause.
+     *
+     * Use of this method should usually be within a conditional check against DB::get_conn()->supportsCteQueries().
+     *
+     * @param string $name The name of the WITH clause, which can be referenced in any queries UNIONed to the $query
+     * and in this query directly, as though it were a table name.
+     * @param string[] $cteFields Aliases for any columns selected in $query which can be referenced in any queries
+     * UNIONed to the $query and in this query directly, as though they were columns in a real table.
+     */
+    public function addWith(string $name, self $query, array $cteFields = [], bool $recursive = false): static
+    {
+        if (array_key_exists($name, $this->with)) {
+            throw new LogicException("WITH clause with name '$name' already exists.");
+        }
+        $this->with[$name] = [
+            'cte_fields' => $cteFields,
+            'query' => $query,
+            'recursive' => $recursive,
+        ];
+        return $this;
+    }
+
+    /**
+     * Get the data which will be used to generate the WITH clause of the query
+     */
+    public function getWith(): array
+    {
+        return $this->with;
+    }
+
     /**
      * Return a list of GROUP BY clauses used internally.
      *