Skip to content

Commit

Permalink
supplement the doc and rename the store-key
Browse files Browse the repository at this point in the history
  • Loading branch information
sunxiaojian committed Mar 6, 2024
1 parent 0b7b739 commit bd81c01
Show file tree
Hide file tree
Showing 7 changed files with 50 additions and 48 deletions.
10 changes: 6 additions & 4 deletions docs/content/how-to/creating-catalogs.md
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,9 @@ For instance, using the option `hive.table.owner=Jon` will automatically add the
## Creating a Catalog with JDBC Metastore
By using the Paimon JDBC catalog, changes to the catalog will be directly stored in relational databases such as MySQL, postgres, etc.
By using the Paimon JDBC catalog, changes to the catalog will be directly stored in relational databases such as SQLite, MySQL, postgres, etc.
Currently, lock configuration is only supported for MySQL and SQLite. If you are using a different type of database for catalog storage, please do not configure `lock.enabled`.
{{< tabs "jdbc-metastore-example" >}}
Expand All @@ -198,15 +200,15 @@ CREATE CATALOG my_jdbc WITH (
'uri' = 'jdbc:mysql://<host>:<port>/<databaseName>',
'jdbc.user' = '...',
'jdbc.password' = '...',
'store-key'='jdbc',
'catalog-key'='jdbc',
'warehouse' = 'hdfs:///path/to/warehouse'
);
USE CATALOG my_jdbc;
```
You can define any connection parameters for a database with the prefix "jdbc.".
You can configure any connection parameters that have been declared by JDBC through "jdbc.", the connection parameters may be different between different databases, please configure according to the actual situation.
You can also perform logical isolation for databases under multiple catalogs by specifying "store-key".
You can also perform logical isolation for databases under multiple catalogs by specifying "catalog-key".
You can define any default table options with the prefix `table-default.` for tables created in the catalog.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
</thead>
<tbody>
<tr>
<td><h5>store-key</h5></td>
<td><h5>catalog-key</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
<td>Custom jdbc catalog store key.</td>
Expand Down
38 changes: 19 additions & 19 deletions paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,15 @@ public class JdbcCatalog extends AbstractCatalog {
public static final String PROPERTY_PREFIX = "jdbc.";
private static final String DATABASE_EXISTS_PROPERTY = "exists";
private JdbcClientPool connections;
private String storeKey = "jdbc";
private String catalogKey = "jdbc";
private Map<String, String> configuration;
private final String warehouse;

protected JdbcCatalog(
FileIO fileIO, String storeKey, Map<String, String> config, String warehouse) {
FileIO fileIO, String catalogKey, Map<String, String> config, String warehouse) {
super(fileIO);
if (!StringUtils.isBlank(storeKey)) {
this.storeKey = storeKey;
if (!StringUtils.isBlank(catalogKey)) {
this.catalogKey = catalogKey;
}
this.configuration = config;
this.warehouse = warehouse;
Expand Down Expand Up @@ -148,19 +148,19 @@ public List<String> listDatabases() {
fetch(
row -> row.getString(JdbcUtils.TABLE_DATABASE),
JdbcUtils.LIST_ALL_TABLE_DATABASES_SQL,
storeKey));
catalogKey));

databases.addAll(
fetch(
row -> row.getString(JdbcUtils.DATABASE_NAME),
JdbcUtils.LIST_ALL_PROPERTY_DATABASES_SQL,
storeKey));
catalogKey));
return databases;
}

@Override
protected boolean databaseExistsImpl(String databaseName) {
return JdbcUtils.databaseExists(connections, storeKey, databaseName);
return JdbcUtils.databaseExists(connections, catalogKey, databaseName);
}

@Override
Expand Down Expand Up @@ -193,15 +193,15 @@ protected void createDatabaseImpl(String name, Map<String, String> properties) {
Path databasePath = newDatabasePath(name);
createProps.put(DB_LOCATION_PROP, databasePath.toString());
}
insertProperties(connections, storeKey, name, createProps);
insertProperties(connections, catalogKey, name, createProps);
}

@Override
protected void dropDatabaseImpl(String name) {
// Delete table from paimon_tables
execute(connections, JdbcUtils.DELETE_TABLES_SQL, storeKey, name);
execute(connections, JdbcUtils.DELETE_TABLES_SQL, catalogKey, name);
// Delete properties from paimon_database_properties
execute(connections, JdbcUtils.DELETE_ALL_DATABASE_PROPERTIES_SQL, storeKey, name);
execute(connections, JdbcUtils.DELETE_ALL_DATABASE_PROPERTIES_SQL, catalogKey, name);
}

@Override
Expand All @@ -212,7 +212,7 @@ protected List<String> listTablesImpl(String databaseName) {
return fetch(
row -> row.getString(JdbcUtils.TABLE_NAME),
JdbcUtils.LIST_TABLES_SQL,
storeKey,
catalogKey,
databaseName);
}

Expand All @@ -223,7 +223,7 @@ protected void dropTableImpl(Identifier identifier) {
execute(
connections,
JdbcUtils.DROP_TABLE_SQL,
storeKey,
catalogKey,
identifier.getDatabaseName(),
identifier.getObjectName());

Expand Down Expand Up @@ -257,7 +257,7 @@ protected void createTableImpl(Identifier identifier, Schema schema) {
try (PreparedStatement sql =
conn.prepareStatement(
JdbcUtils.DO_COMMIT_CREATE_TABLE_SQL)) {
sql.setString(1, storeKey);
sql.setString(1, catalogKey);
sql.setString(2, identifier.getDatabaseName());
sql.setString(3, identifier.getObjectName());
return sql.executeUpdate();
Expand All @@ -274,7 +274,7 @@ protected void createTableImpl(Identifier identifier, Schema schema) {
throw new RuntimeException(
String.format(
"Failed to create table %s in catalog %s",
identifier.getFullName(), storeKey));
identifier.getFullName(), catalogKey));
}
} catch (Exception e) {
throw new RuntimeException("Failed to create table " + identifier.getFullName(), e);
Expand All @@ -285,7 +285,7 @@ protected void createTableImpl(Identifier identifier, Schema schema) {
protected void renameTableImpl(Identifier fromTable, Identifier toTable) {
try {
// update table metadata info
updateTable(connections, storeKey, fromTable, toTable);
updateTable(connections, catalogKey, fromTable, toTable);

Path fromPath = getDataTableLocation(fromTable);
if (new SchemaManager(fileIO, fromPath).listAllIds().size() > 0) {
Expand Down Expand Up @@ -336,7 +336,7 @@ public boolean tableExists(Identifier identifier) {
return super.tableExists(identifier);
}
return JdbcUtils.tableExists(
connections, storeKey, identifier.getDatabaseName(), identifier.getObjectName());
connections, catalogKey, identifier.getDatabaseName(), identifier.getObjectName());
}

@Override
Expand All @@ -347,7 +347,7 @@ public boolean caseSensitive() {
@Override
public Optional<CatalogLock.LockFactory> lockFactory() {
return lockEnabled()
? Optional.of(JdbcCatalogLock.createFactory(connections, storeKey, configuration))
? Optional.of(JdbcCatalogLock.createFactory(connections, catalogKey, configuration))
: Optional.empty();
}

Expand All @@ -364,7 +364,7 @@ private Lock lock(Identifier identifier) {
JdbcCatalogLock lock =
new JdbcCatalogLock(
connections,
storeKey,
catalogKey,
checkMaxSleep(configuration),
acquireTimeout(configuration));
return Lock.fromCatalog(lock, identifier);
Expand Down Expand Up @@ -393,7 +393,7 @@ private Map<String, String> fetchProperties(String databaseName) {
row.getString(JdbcUtils.DATABASE_PROPERTY_KEY),
row.getString(JdbcUtils.DATABASE_PROPERTY_VALUE)),
JdbcUtils.GET_ALL_DATABASE_PROPERTIES_SQL,
storeKey,
catalogKey,
databaseName);
return ImmutableMap.<String, String>builder().putAll(entries).build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public String identifier() {

@Override
public Catalog create(FileIO fileIO, Path warehouse, CatalogContext context) {
String storeKey = context.options().get(JdbcCatalogOptions.STORE_KEY);
return new JdbcCatalog(fileIO, storeKey, context.options().toMap(), warehouse.getName());
String catalogKey = context.options().get(JdbcCatalogOptions.CATALOG_KEY);
return new JdbcCatalog(fileIO, catalogKey, context.options().toMap(), warehouse.getName());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
/** Options for jdbc catalog. */
public final class JdbcCatalogOptions {

public static final ConfigOption<String> STORE_KEY =
ConfigOptions.key("store-key")
public static final ConfigOption<String> CATALOG_KEY =
ConfigOptions.key("catalog-key")
.stringType()
.defaultValue(null)
.withDescription("Custom jdbc catalog store key.");
Expand Down
38 changes: 19 additions & 19 deletions paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,22 @@
public class JdbcUtils {
private static final Logger LOG = LoggerFactory.getLogger(JdbcUtils.class);
public static final String CATALOG_TABLE_NAME = "paimon_tables";
public static final String STORE_KEY = "store_key";
public static final String CATALOG_KEY = "catalog_key";
public static final String TABLE_DATABASE = "database_name";
public static final String TABLE_NAME = "table_name";

static final String CREATE_CATALOG_TABLE =
"CREATE TABLE "
+ CATALOG_TABLE_NAME
+ "("
+ STORE_KEY
+ CATALOG_KEY
+ " VARCHAR(255) NOT NULL,"
+ TABLE_DATABASE
+ " VARCHAR(255) NOT NULL,"
+ TABLE_NAME
+ " VARCHAR(255) NOT NULL,"
+ " PRIMARY KEY ("
+ STORE_KEY
+ CATALOG_KEY
+ ", "
+ TABLE_DATABASE
+ ", "
Expand All @@ -64,7 +64,7 @@ public class JdbcUtils {
"SELECT * FROM "
+ CATALOG_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ TABLE_DATABASE
+ " = ? AND "
Expand All @@ -74,7 +74,7 @@ public class JdbcUtils {
"SELECT * FROM "
+ CATALOG_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ TABLE_DATABASE
+ " = ?";
Expand All @@ -83,7 +83,7 @@ public class JdbcUtils {
"DELETE FROM "
+ CATALOG_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ TABLE_DATABASE
+ " = ?";
Expand All @@ -96,7 +96,7 @@ public class JdbcUtils {
+ TABLE_NAME
+ " = ? "
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ TABLE_DATABASE
+ " = ? AND "
Expand All @@ -106,7 +106,7 @@ public class JdbcUtils {
"DELETE FROM "
+ CATALOG_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ TABLE_DATABASE
+ " = ? AND "
Expand All @@ -118,7 +118,7 @@ public class JdbcUtils {
+ " FROM "
+ CATALOG_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ TABLE_DATABASE
+ " = ? LIMIT 1";
Expand All @@ -129,13 +129,13 @@ public class JdbcUtils {
+ " FROM "
+ CATALOG_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ?";
static final String DO_COMMIT_CREATE_TABLE_SQL =
"INSERT INTO "
+ CATALOG_TABLE_NAME
+ " ("
+ STORE_KEY
+ CATALOG_KEY
+ ", "
+ TABLE_DATABASE
+ ", "
Expand All @@ -153,7 +153,7 @@ public class JdbcUtils {
"CREATE TABLE "
+ DATABASE_PROPERTIES_TABLE_NAME
+ "("
+ STORE_KEY
+ CATALOG_KEY
+ " VARCHAR(255) NOT NULL,"
+ DATABASE_NAME
+ " VARCHAR(255) NOT NULL,"
Expand All @@ -162,7 +162,7 @@ public class JdbcUtils {
+ DATABASE_PROPERTY_VALUE
+ " VARCHAR(1000),"
+ "PRIMARY KEY ("
+ STORE_KEY
+ CATALOG_KEY
+ ", "
+ DATABASE_NAME
+ ", "
Expand All @@ -175,15 +175,15 @@ public class JdbcUtils {
+ " FROM "
+ DATABASE_PROPERTIES_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ DATABASE_NAME
+ " = ? ";
static final String INSERT_DATABASE_PROPERTIES_SQL =
"INSERT INTO "
+ DATABASE_PROPERTIES_TABLE_NAME
+ " ("
+ STORE_KEY
+ CATALOG_KEY
+ ", "
+ DATABASE_NAME
+ ", "
Expand All @@ -197,15 +197,15 @@ public class JdbcUtils {
+ " FROM "
+ DATABASE_PROPERTIES_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ DATABASE_NAME
+ " = ? ";
static final String DELETE_ALL_DATABASE_PROPERTIES_SQL =
"DELETE FROM "
+ DATABASE_PROPERTIES_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ? AND "
+ DATABASE_NAME
+ " = ?";
Expand All @@ -215,7 +215,7 @@ public class JdbcUtils {
+ " FROM "
+ DATABASE_PROPERTIES_TABLE_NAME
+ " WHERE "
+ STORE_KEY
+ CATALOG_KEY
+ " = ?";

// Distributed locks table
Expand Down Expand Up @@ -250,7 +250,7 @@ public static Map<String, String> getTable(
sql.setString(3, tableName);
ResultSet rs = sql.executeQuery();
if (rs.next()) {
table.put(STORE_KEY, rs.getString(STORE_KEY));
table.put(CATALOG_KEY, rs.getString(CATALOG_KEY));
table.put(TABLE_DATABASE, rs.getString(TABLE_DATABASE));
table.put(TABLE_NAME, rs.getString(TABLE_NAME));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

package org.apache.paimon.jdbc;

/** Distributed lock implementation based on sqlite table. */
/** Distributed lock implementation based on mysql table. */
public class MysqlDistributedLockDialect extends AbstractDistributedLockDialect {

@Override
Expand Down

0 comments on commit bd81c01

Please sign in to comment.