diff --git a/docs/content/how-to/creating-catalogs.md b/docs/content/how-to/creating-catalogs.md index 9c7dce1b7370..43d23dc1f51a 100644 --- a/docs/content/how-to/creating-catalogs.md +++ b/docs/content/how-to/creating-catalogs.md @@ -178,7 +178,9 @@ For instance, using the option `hive.table.owner=Jon` will automatically add the ## Creating a Catalog with JDBC Metastore -By using the Paimon JDBC catalog, changes to the catalog will be directly stored in relational databases such as MySQL, postgres, etc. +By using the Paimon JDBC catalog, changes to the catalog will be directly stored in relational databases such as SQLite, MySQL, postgres, etc. + +Currently, lock configuration is only supported for MySQL and SQLite. If you are using a different type of database for catalog storage, please do not configure `lock.enabled`. {{< tabs "jdbc-metastore-example" >}} @@ -198,15 +200,15 @@ CREATE CATALOG my_jdbc WITH ( 'uri' = 'jdbc:mysql://:/', 'jdbc.user' = '...', 'jdbc.password' = '...', - 'store-key'='jdbc', + 'catalog-key'='jdbc', 'warehouse' = 'hdfs:///path/to/warehouse' ); USE CATALOG my_jdbc; ``` -You can define any connection parameters for a database with the prefix "jdbc.". +You can configure any connection parameters that have been declared by JDBC through "jdbc.", the connection parameters may be different between different databases, please configure according to the actual situation. -You can also perform logical isolation for databases under multiple catalogs by specifying "store-key". +You can also perform logical isolation for databases under multiple catalogs by specifying "catalog-key". You can define any default table options with the prefix `table-default.` for tables created in the catalog. diff --git a/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html b/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html index 0cad5380d7d8..617018ce9f5b 100644 --- a/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html +++ b/docs/layouts/shortcodes/generated/jdbc_catalog_configuration.html @@ -27,7 +27,7 @@ -
store-key
+
catalog-key
(none) String Custom jdbc catalog store key. diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java index f1eb5f66d8cb..5dc2abc9e416 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java @@ -65,15 +65,15 @@ public class JdbcCatalog extends AbstractCatalog { public static final String PROPERTY_PREFIX = "jdbc."; private static final String DATABASE_EXISTS_PROPERTY = "exists"; private JdbcClientPool connections; - private String storeKey = "jdbc"; + private String catalogKey = "jdbc"; private Map configuration; private final String warehouse; protected JdbcCatalog( - FileIO fileIO, String storeKey, Map config, String warehouse) { + FileIO fileIO, String catalogKey, Map config, String warehouse) { super(fileIO); - if (!StringUtils.isBlank(storeKey)) { - this.storeKey = storeKey; + if (!StringUtils.isBlank(catalogKey)) { + this.catalogKey = catalogKey; } this.configuration = config; this.warehouse = warehouse; @@ -148,19 +148,19 @@ public List listDatabases() { fetch( row -> row.getString(JdbcUtils.TABLE_DATABASE), JdbcUtils.LIST_ALL_TABLE_DATABASES_SQL, - storeKey)); + catalogKey)); databases.addAll( fetch( row -> row.getString(JdbcUtils.DATABASE_NAME), JdbcUtils.LIST_ALL_PROPERTY_DATABASES_SQL, - storeKey)); + catalogKey)); return databases; } @Override protected boolean databaseExistsImpl(String databaseName) { - return JdbcUtils.databaseExists(connections, storeKey, databaseName); + return JdbcUtils.databaseExists(connections, catalogKey, databaseName); } @Override @@ -193,15 +193,15 @@ protected void createDatabaseImpl(String name, Map properties) { Path databasePath = newDatabasePath(name); createProps.put(DB_LOCATION_PROP, databasePath.toString()); } - insertProperties(connections, storeKey, name, createProps); + insertProperties(connections, catalogKey, name, createProps); } @Override protected void dropDatabaseImpl(String name) { // Delete table from paimon_tables - execute(connections, JdbcUtils.DELETE_TABLES_SQL, storeKey, name); + execute(connections, JdbcUtils.DELETE_TABLES_SQL, catalogKey, name); // Delete properties from paimon_database_properties - execute(connections, JdbcUtils.DELETE_ALL_DATABASE_PROPERTIES_SQL, storeKey, name); + execute(connections, JdbcUtils.DELETE_ALL_DATABASE_PROPERTIES_SQL, catalogKey, name); } @Override @@ -212,7 +212,7 @@ protected List listTablesImpl(String databaseName) { return fetch( row -> row.getString(JdbcUtils.TABLE_NAME), JdbcUtils.LIST_TABLES_SQL, - storeKey, + catalogKey, databaseName); } @@ -223,7 +223,7 @@ protected void dropTableImpl(Identifier identifier) { execute( connections, JdbcUtils.DROP_TABLE_SQL, - storeKey, + catalogKey, identifier.getDatabaseName(), identifier.getObjectName()); @@ -257,7 +257,7 @@ protected void createTableImpl(Identifier identifier, Schema schema) { try (PreparedStatement sql = conn.prepareStatement( JdbcUtils.DO_COMMIT_CREATE_TABLE_SQL)) { - sql.setString(1, storeKey); + sql.setString(1, catalogKey); sql.setString(2, identifier.getDatabaseName()); sql.setString(3, identifier.getObjectName()); return sql.executeUpdate(); @@ -274,7 +274,7 @@ protected void createTableImpl(Identifier identifier, Schema schema) { throw new RuntimeException( String.format( "Failed to create table %s in catalog %s", - identifier.getFullName(), storeKey)); + identifier.getFullName(), catalogKey)); } } catch (Exception e) { throw new RuntimeException("Failed to create table " + identifier.getFullName(), e); @@ -285,7 +285,7 @@ protected void createTableImpl(Identifier identifier, Schema schema) { protected void renameTableImpl(Identifier fromTable, Identifier toTable) { try { // update table metadata info - updateTable(connections, storeKey, fromTable, toTable); + updateTable(connections, catalogKey, fromTable, toTable); Path fromPath = getDataTableLocation(fromTable); if (new SchemaManager(fileIO, fromPath).listAllIds().size() > 0) { @@ -336,7 +336,7 @@ public boolean tableExists(Identifier identifier) { return super.tableExists(identifier); } return JdbcUtils.tableExists( - connections, storeKey, identifier.getDatabaseName(), identifier.getObjectName()); + connections, catalogKey, identifier.getDatabaseName(), identifier.getObjectName()); } @Override @@ -347,7 +347,7 @@ public boolean caseSensitive() { @Override public Optional lockFactory() { return lockEnabled() - ? Optional.of(JdbcCatalogLock.createFactory(connections, storeKey, configuration)) + ? Optional.of(JdbcCatalogLock.createFactory(connections, catalogKey, configuration)) : Optional.empty(); } @@ -364,7 +364,7 @@ private Lock lock(Identifier identifier) { JdbcCatalogLock lock = new JdbcCatalogLock( connections, - storeKey, + catalogKey, checkMaxSleep(configuration), acquireTimeout(configuration)); return Lock.fromCatalog(lock, identifier); @@ -393,7 +393,7 @@ private Map fetchProperties(String databaseName) { row.getString(JdbcUtils.DATABASE_PROPERTY_KEY), row.getString(JdbcUtils.DATABASE_PROPERTY_VALUE)), JdbcUtils.GET_ALL_DATABASE_PROPERTIES_SQL, - storeKey, + catalogKey, databaseName); return ImmutableMap.builder().putAll(entries).build(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogFactory.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogFactory.java index 311125a48b5f..1c791eb5e11e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogFactory.java @@ -36,7 +36,7 @@ public String identifier() { @Override public Catalog create(FileIO fileIO, Path warehouse, CatalogContext context) { - String storeKey = context.options().get(JdbcCatalogOptions.STORE_KEY); - return new JdbcCatalog(fileIO, storeKey, context.options().toMap(), warehouse.getName()); + String catalogKey = context.options().get(JdbcCatalogOptions.CATALOG_KEY); + return new JdbcCatalog(fileIO, catalogKey, context.options().toMap(), warehouse.getName()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java index 0764ff93dda2..dd4afd473e22 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalogOptions.java @@ -24,8 +24,8 @@ /** Options for jdbc catalog. */ public final class JdbcCatalogOptions { - public static final ConfigOption STORE_KEY = - ConfigOptions.key("store-key") + public static final ConfigOption CATALOG_KEY = + ConfigOptions.key("catalog-key") .stringType() .defaultValue(null) .withDescription("Custom jdbc catalog store key."); diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcUtils.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcUtils.java index de205b680aca..7b9b93a5a4e2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcUtils.java @@ -38,7 +38,7 @@ public class JdbcUtils { private static final Logger LOG = LoggerFactory.getLogger(JdbcUtils.class); public static final String CATALOG_TABLE_NAME = "paimon_tables"; - public static final String STORE_KEY = "store_key"; + public static final String CATALOG_KEY = "catalog_key"; public static final String TABLE_DATABASE = "database_name"; public static final String TABLE_NAME = "table_name"; @@ -46,14 +46,14 @@ public class JdbcUtils { "CREATE TABLE " + CATALOG_TABLE_NAME + "(" - + STORE_KEY + + CATALOG_KEY + " VARCHAR(255) NOT NULL," + TABLE_DATABASE + " VARCHAR(255) NOT NULL," + TABLE_NAME + " VARCHAR(255) NOT NULL," + " PRIMARY KEY (" - + STORE_KEY + + CATALOG_KEY + ", " + TABLE_DATABASE + ", " @@ -64,7 +64,7 @@ public class JdbcUtils { "SELECT * FROM " + CATALOG_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + TABLE_DATABASE + " = ? AND " @@ -74,7 +74,7 @@ public class JdbcUtils { "SELECT * FROM " + CATALOG_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + TABLE_DATABASE + " = ?"; @@ -83,7 +83,7 @@ public class JdbcUtils { "DELETE FROM " + CATALOG_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + TABLE_DATABASE + " = ?"; @@ -96,7 +96,7 @@ public class JdbcUtils { + TABLE_NAME + " = ? " + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + TABLE_DATABASE + " = ? AND " @@ -106,7 +106,7 @@ public class JdbcUtils { "DELETE FROM " + CATALOG_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + TABLE_DATABASE + " = ? AND " @@ -118,7 +118,7 @@ public class JdbcUtils { + " FROM " + CATALOG_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + TABLE_DATABASE + " = ? LIMIT 1"; @@ -129,13 +129,13 @@ public class JdbcUtils { + " FROM " + CATALOG_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ?"; static final String DO_COMMIT_CREATE_TABLE_SQL = "INSERT INTO " + CATALOG_TABLE_NAME + " (" - + STORE_KEY + + CATALOG_KEY + ", " + TABLE_DATABASE + ", " @@ -153,7 +153,7 @@ public class JdbcUtils { "CREATE TABLE " + DATABASE_PROPERTIES_TABLE_NAME + "(" - + STORE_KEY + + CATALOG_KEY + " VARCHAR(255) NOT NULL," + DATABASE_NAME + " VARCHAR(255) NOT NULL," @@ -162,7 +162,7 @@ public class JdbcUtils { + DATABASE_PROPERTY_VALUE + " VARCHAR(1000)," + "PRIMARY KEY (" - + STORE_KEY + + CATALOG_KEY + ", " + DATABASE_NAME + ", " @@ -175,7 +175,7 @@ public class JdbcUtils { + " FROM " + DATABASE_PROPERTIES_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + DATABASE_NAME + " = ? "; @@ -183,7 +183,7 @@ public class JdbcUtils { "INSERT INTO " + DATABASE_PROPERTIES_TABLE_NAME + " (" - + STORE_KEY + + CATALOG_KEY + ", " + DATABASE_NAME + ", " @@ -197,7 +197,7 @@ public class JdbcUtils { + " FROM " + DATABASE_PROPERTIES_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + DATABASE_NAME + " = ? "; @@ -205,7 +205,7 @@ public class JdbcUtils { "DELETE FROM " + DATABASE_PROPERTIES_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ? AND " + DATABASE_NAME + " = ?"; @@ -215,7 +215,7 @@ public class JdbcUtils { + " FROM " + DATABASE_PROPERTIES_TABLE_NAME + " WHERE " - + STORE_KEY + + CATALOG_KEY + " = ?"; // Distributed locks table @@ -250,7 +250,7 @@ public static Map getTable( sql.setString(3, tableName); ResultSet rs = sql.executeQuery(); if (rs.next()) { - table.put(STORE_KEY, rs.getString(STORE_KEY)); + table.put(CATALOG_KEY, rs.getString(CATALOG_KEY)); table.put(TABLE_DATABASE, rs.getString(TABLE_DATABASE)); table.put(TABLE_NAME, rs.getString(TABLE_NAME)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/MysqlDistributedLockDialect.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/MysqlDistributedLockDialect.java index 903108762cc7..206aa8cd77ad 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/MysqlDistributedLockDialect.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/MysqlDistributedLockDialect.java @@ -18,7 +18,7 @@ package org.apache.paimon.jdbc; -/** Distributed lock implementation based on sqlite table. */ +/** Distributed lock implementation based on mysql table. */ public class MysqlDistributedLockDialect extends AbstractDistributedLockDialect { @Override