Skip to content

Commit

Permalink
Apply Table Import (#116)
Browse files Browse the repository at this point in the history
* Table import
  • Loading branch information
akageun authored Sep 23, 2024
1 parent 74d84c6 commit 5df5cf7
Show file tree
Hide file tree
Showing 14 changed files with 512 additions and 49 deletions.
3 changes: 2 additions & 1 deletion cassdio-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@ dependencies {
// Json
implementation("org.springframework.boot:spring-boot-starter-json")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.1")

implementation("org.apache.commons:commons-csv:1.9.0")
// CommonsLang3
api("org.apache.commons:commons-lang3:3.13.0")
api("com.google.guava:guava:33.0.0-jre")
api("org.apache.commons:commons-collections4:4.4")
api("org.apache.commons:commons-csv:1.9.0")

// Cache
implementation("org.springframework.boot:spring-boot-starter-cache")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package kr.hakdang.cassdio.common.utils;

/**
* CsvHelper
*
* @author akageun
* @since 2024-08-19
*/
public class CsvHelper {
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package kr.hakdang.cassdio.core.domain.cluster.keyspace.table;

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* ClusterTableCsvProvider
*
* @author akageun
* @since 2024-08-08
*/
@Slf4j
@Service
public class ClusterCsvProvider {

public void importerCsvSampleDownload(Writer writer, List<String> headerList) {
CSVFormat csvFormat = CSVFormat.DEFAULT.builder()
.setHeader(headerList.toArray(String[]::new))
.build();

try (final CSVPrinter printer = new CSVPrinter(writer, csvFormat)) {
log.info("create complete importer csv sample");

printer.flush();
} catch (IOException e) {
throw new RuntimeException(e);
}

}

public List<Map<String, Object>> importCsvReader(Reader reader, List<String> columnList) throws IOException {
CSVFormat csvFormat = CSVFormat.DEFAULT.builder()
.setHeader(columnList.toArray(String[]::new))
.setSkipHeaderRecord(true)
.setTrim(true)
.build();

Iterable<CSVRecord> records = csvFormat.parse(reader);
//Validation 방식에 대해 고민 필요

List<Map<String, Object>> values = new ArrayList<>();

for (CSVRecord record : records) {

Map<String, Object> map = new HashMap<>();

for (String column : columnList) {
map.put(column, StringUtils.defaultIfBlank(record.get(column), ""));
}

values.add(map);
}

return values;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package kr.hakdang.cassdio.core.domain.cluster.keyspace.table;

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.io.Writer;
import java.util.List;

/**
* ClusterTableCsvProvider
*
* @author akageun
* @since 2024-08-08
*/
@Slf4j
@Service
public class ClusterTableCsvProvider {

public void importerCsvSampleDownload(Writer writer, List<String> sortedColumnList) {
CSVFormat csvFormat = CSVFormat.DEFAULT.builder()
.setHeader(sortedColumnList.toArray(String[]::new))
.build();

try (final CSVPrinter printer = new CSVPrinter(writer, csvFormat)) {
//printer.printRecord(author, title);
log.info("create complete importer csv sample");
} catch (IOException e) {
throw new RuntimeException(e);
}
}


}
Original file line number Diff line number Diff line change
@@ -1,18 +1,27 @@
package kr.hakdang.cassdio.core.domain.cluster.keyspace.table;

import com.datastax.oss.driver.api.core.CqlSession;
import com.datastax.oss.driver.api.core.cql.BatchStatement;
import com.datastax.oss.driver.api.core.cql.DefaultBatchType;
import com.datastax.oss.driver.api.core.cql.ResultSet;
import com.datastax.oss.driver.api.core.cql.SimpleStatement;
import com.datastax.oss.driver.api.querybuilder.QueryBuilder;
import com.datastax.oss.driver.api.querybuilder.insert.InsertInto;
import com.datastax.oss.driver.api.querybuilder.insert.JsonInsert;
import com.datastax.oss.protocol.internal.util.Bytes;
import com.google.common.collect.Lists;
import kr.hakdang.cassdio.common.utils.Jsons;
import kr.hakdang.cassdio.core.domain.cluster.BaseClusterCommander;
import kr.hakdang.cassdio.core.domain.cluster.CqlSessionSelectResults;
import kr.hakdang.cassdio.core.domain.cluster.keyspace.CassdioColumnDefinition;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;

import java.time.Duration;
import java.util.List;
import java.util.Map;

/**
* ClusterTableRowCommander
Expand Down Expand Up @@ -42,4 +51,26 @@ public CqlSessionSelectResults rowSelect(String clusterId, TableDTO.ClusterTable
resultSet.getExecutionInfo().getPagingState()
);
}

public void rowInserts(TableDTO.ClusterTableRowImportArgs args, List<Map<String, Object>> values) {
if (CollectionUtils.isEmpty(values)) {
return;
}

CqlSession session = cqlSessionFactory.get(args.getClusterId());

for (List<Map<String, Object>> list : Lists.partition(values, args.getPerCommitSize())) {
BatchStatement batchStatement = BatchStatement.newInstance(args.getBatchType());

for (Map<String, Object> map : list) {
batchStatement = batchStatement.add(
QueryBuilder.insertInto(args.getKeyspace(), args.getTable())
.json(Jsons.toJson(map))
.build()
);
}

session.execute(batchStatement);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package kr.hakdang.cassdio.core.domain.cluster.keyspace.table;

import com.datastax.oss.driver.api.core.cql.BatchType;
import com.datastax.oss.driver.api.core.cql.DefaultBatchType;
import io.micrometer.common.util.StringUtils;
import lombok.AccessLevel;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.apache.commons.lang3.EnumUtils;

/**
* ClusterTableArgs
Expand Down Expand Up @@ -99,4 +102,31 @@ public ClusterTableRowArgs(String keyspace, String table, int pageSize, int time
}
}

@ToString
@Getter
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public static class ClusterTableRowImportArgs {
private String clusterId;
private String keyspace;
private String table;

private BatchType batchType;
private int perCommitSize = 50;

@Builder
public ClusterTableRowImportArgs(
String clusterId,
String keyspace,
String table,
String batchTypeCode,
int perCommitSize
) {
this.clusterId = clusterId;
this.keyspace = keyspace;
this.table = table;
this.batchType = EnumUtils.getEnum(DefaultBatchType.class, batchTypeCode);
this.perCommitSize = perCommitSize;
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@
import kr.hakdang.cassdio.core.domain.cluster.keyspace.CassdioColumnDefinition;
import kr.hakdang.cassdio.core.domain.cluster.keyspace.table.CassandraSystemTable;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;

import java.time.Duration;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker;
import static java.util.Collections.emptyList;
Expand All @@ -40,7 +40,6 @@ public CqlSessionSelectResults columnList(String clusterId, String keyspace, Str

public CqlSessionSelectResults columnList(String clusterId, String keyspace, String table, List<String> columnList) {
CqlSession session = cqlSessionFactory.get(clusterId);

SimpleStatement statement;

Select select = getColumnTable(session, keyspace)
Expand Down Expand Up @@ -69,6 +68,11 @@ public CqlSessionSelectResults columnList(String clusterId, String keyspace, Str
);
}

public List<String> columnSortedList(String clusterId, String keyspace, String table) {
CqlSessionSelectResults results = columnList(clusterId, keyspace, table);
return results.getRows().stream().map(row -> String.valueOf(row.get("column_name"))).collect(Collectors.toList());
}

private String makeSortValue(Map<String, Object> row) {
ColumnKind columnKind = ColumnKind.findByCode(String.valueOf(row.get("kind")));
return String.format("%s-%s", columnKind.getOrder(), row.get("position"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import jakarta.validation.Valid;
import kr.hakdang.cassdio.core.domain.cluster.CqlSessionSelectResult;
import kr.hakdang.cassdio.core.domain.cluster.CqlSessionSelectResults;
import kr.hakdang.cassdio.core.domain.cluster.keyspace.table.ClusterCsvProvider;
import kr.hakdang.cassdio.core.domain.cluster.keyspace.table.ClusterTableCommander;
import kr.hakdang.cassdio.core.domain.cluster.keyspace.table.ClusterTableGetCommander;
import kr.hakdang.cassdio.core.domain.cluster.keyspace.table.ClusterTableListCommander;
Expand All @@ -13,7 +14,6 @@
import kr.hakdang.cassdio.web.common.dto.response.ApiResponse;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
Expand All @@ -33,20 +33,19 @@ public class ClusterTableApi {
private final ClusterTableCommander clusterTableCommander;
private final ClusterTableListCommander clusterTableListCommander;
private final ClusterTableGetCommander clusterTableGetCommander;
private final ClusterTableRowCommander clusterTableRowCommander;

private final ClusterTableColumnCommander clusterTableColumnCommander;


public ClusterTableApi(
ClusterTableCommander clusterTableCommander,
ClusterTableListCommander clusterTableListCommander,
ClusterTableGetCommander clusterTableGetCommander,
ClusterTableRowCommander clusterTableRowCommander,
ClusterTableColumnCommander clusterTableColumnCommander
) {
this.clusterTableCommander = clusterTableCommander;
this.clusterTableListCommander = clusterTableListCommander;
this.clusterTableGetCommander = clusterTableGetCommander;
this.clusterTableRowCommander = clusterTableRowCommander;
this.clusterTableColumnCommander = clusterTableColumnCommander;
}

Expand Down Expand Up @@ -106,25 +105,6 @@ public ApiResponse<Map<String, Object>> getTableColumn(
return ApiResponse.ok(responseMap);
}

@GetMapping("/table/{table}/row")
public ApiResponse<Map<String, Object>> tableRow(
@PathVariable String clusterId,
@PathVariable String keyspace,
@PathVariable String table,
@ModelAttribute ClusterTableRowRequest request
) {
Map<String, Object> responseMap = new HashMap<>();
CqlSessionSelectResults result1 = clusterTableRowCommander.rowSelect(clusterId, request.makeArgs(keyspace, table));

responseMap.put("nextCursor", result1.getNextCursor());
responseMap.put("rows", result1.getRows());
responseMap.put("rowHeader", result1.getRowHeader());

responseMap.put("columnList", clusterTableColumnCommander.columnList(clusterId, keyspace, table));

return ApiResponse.ok(responseMap);
}

//권한 추가해서 ADMIN 만 동작할 수 있도록 해야함.
@DeleteMapping("/table/{table}/drop")
public ApiResponse<Void> tableDrop(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package kr.hakdang.cassdio.web.route.cluster.keyspace.table;

import lombok.AccessLevel;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;

/**
* ClusterTableRequest
*
* @author akageun
* @since 2024-08-19
*/
public class ClusterTableRequest {

@Getter
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public static class TableRowImportRequest {
private int perCommitSize;
private String batchTypeCode;

@Builder
public TableRowImportRequest(int perCommitSize, String batchTypeCode) {
this.perCommitSize = perCommitSize;
this.batchTypeCode = batchTypeCode;
}
}
}
Loading

0 comments on commit 5df5cf7

Please sign in to comment.