Skip to content

Commit

Permalink
[fix](regression-test) Make test_decimal256_outfile_csv export the …
Browse files Browse the repository at this point in the history 
…data to S3 rather than local file system #42211 (#42271)

cherry pick from #42211

Co-authored-by: Tiewei Fang <[email protected]>
  • Loading branch information
@morningman @BePPPower
morningman and BePPPower authored Oct 22, 2024
1 parent 53ee740 commit 864bbd7
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 50 deletions.
21 changes: 21 additions & 0 deletions regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,24 @@
4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000

-- !select_tvf1 --
\N \N \N
-9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
-4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
-99999999999999999999999999999.999999999 1.0000000000 1.00000000000
0.000000000 0.0000000000 0.00000000000
1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
2.000000000 499999999999999999999999999999999999999999999999999999999999999999.9999999999 49999999999999999999999999999999999999999999999999999999999999999.99999999999
3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
6.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999
7.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999
99999999999999999999999999999.999999999 1.0000000000 1.00000000000
4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000

85 changes: 35 additions & 50 deletions regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,36 +22,15 @@ import java.nio.file.Files
import java.nio.file.Paths

suite("test_decimal256_outfile_csv") {
StringBuilder strBuilder = new StringBuilder()
strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
strBuilder.append(" http://" + context.config.feHttpAddress + "/rest/v1/config/fe")

String command = strBuilder.toString()
def process = command.toString().execute()
def code = process.waitFor()
def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
def out = process.getText()
logger.info("Request FE Config: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
def response = parseJson(out.trim())
assertEquals(response.code, 0)
assertEquals(response.msg, "success")
def configJson = response.data.rows
boolean enableOutfileToLocal = false
for (Object conf: configJson) {
assert conf instanceof Map
if (((Map<String, String>) conf).get("Name").toLowerCase() == "enable_outfile_to_local") {
enableOutfileToLocal = ((Map<String, String>) conf).get("Value").toLowerCase() == "true"
}
}
if (!enableOutfileToLocal) {
logger.warn("Please set enable_outfile_to_local to true to run test_outfile")
return
}

sql "set enable_nereids_planner = true;"
sql "set enable_decimal256 = true;"

String ak = getS3AK()
String sk = getS3SK()
String s3_endpoint = getS3Endpoint()
String region = getS3Region()
String bucket = context.config.otherConfigs.get("s3BucketName");

sql "DROP TABLE IF EXISTS `test_decimal256_outfile_csv`"
sql """
CREATE TABLE IF NOT EXISTS `test_decimal256_outfile_csv` (
Expand Down Expand Up @@ -105,31 +84,37 @@ suite("test_decimal256_outfile_csv") {
SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3;
"""

def uuid = UUID.randomUUID().toString()
def outFileNamePrefix = """test_decimal256_outfile_csv_${uuid}_"""
def outFilePath = """/tmp/${outFileNamePrefix}"""
FilenameFilter filter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith(outFileNamePrefix);
}
};

def outFilePath = "${bucket}/outfile/csv/test_decimal256_outfile_csv/exp_"

def outfile_to_S3 = { export_table_name, foramt ->
// select ... into outfile ...
def res = sql """
SELECT * FROM ${export_table_name} t ORDER BY k1
INTO OUTFILE "s3://${outFilePath}"
FORMAT AS ${foramt}
PROPERTIES (
"s3.endpoint" = "${s3_endpoint}",
"s3.region" = "${region}",
"s3.secret_key"="${sk}",
"s3.access_key" = "${ak}"
);
"""
return res[0][3]
}

try {
logger.info("outfile: " + outFilePath)
sql """
SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3 INTO OUTFILE "file://${outFilePath}" properties("column_separator" = ",");
"""
File path = new File("/tmp/")
File[] files = path.listFiles(filter)
assert files.length == 1
List<String> outLines = Files.readAllLines(Paths.get(files[0].getAbsolutePath()), StandardCharsets.UTF_8);
assert outLines.size() == 19
def outfile_url = outfile_to_S3("test_decimal256_outfile_csv", "csv")

qt_select_tvf1 """ SELECT * FROM S3 (
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
"region" = "${region}"
);
"""
} finally {
File path = new File("/tmp/")
if (path.exists()) {
for (File f: path.listFiles(filter)) {
f.delete();
}
}
}
}

0 comments on commit 864bbd7

Please sign in to comment.