Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add BOM to help excel to detect the file is UTF-8 encoded. #65

Merged
merged 4 commits into from
Feb 12, 2024
Merged

Add BOM to help excel to detect the file is UTF-8 encoded. #65

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eefeec8
Add bom at the start of csv to ensure proper encoding
Feb 12, 2024
51699d1
fix
Feb 12, 2024
69ba240
fix unit test
Feb 12, 2024
103f29d
fix comment
Feb 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion ...ain/java/org/gridsuite/sensitivityanalysis/server/service/SensitivityAnalysisService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -190,8 +192,10 @@ public byte[] exportSensitivityResultsAsCsv(UUID resultUuid, SensitivityAnalysis
ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) {
zipOutputStream.putNextEntry(new ZipEntry("sensitivity_result.csv"));

// We add a BOM to indicate that the file is encoded in UTF-8.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please copy the same comment than for SA :

Suggested change
// We add a BOM to indicate that the file is encoded in UTF-8.
// // adding BOM to the beginning of file to help excel in some versions to detect this is UTF-8 encoding bytes

writeUTF8Bom(zipOutputStream);
CsvWriterSettings settings = new CsvWriterSettings();
CsvWriter csvWriter = new CsvWriter(zipOutputStream, settings);
CsvWriter csvWriter = new CsvWriter(zipOutputStream, StandardCharsets.UTF_8, settings);
csvWriter.writeHeaders(sensitivityAnalysisCsvFileInfos.getCsvHeaders());
if (selector.getTabSelection() == ResultTab.N) {
result.getSensitivities()
Expand Down Expand Up @@ -223,6 +227,12 @@ public byte[] exportSensitivityResultsAsCsv(UUID resultUuid, SensitivityAnalysis
}
}

private static void writeUTF8Bom(OutputStream outputStream) throws IOException {
outputStream.write(0xef);
outputStream.write(0xbb);
outputStream.write(0xbf);
}

private static Double nullIfNan(double d) {
return Double.isNaN(d) ? null : d;
}
Expand Down
4 changes: 3 additions & 1 deletion ...test/java/org/gridsuite/sensitivityanalysis/server/SensitivityAnalysisControllerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,9 @@ void runAndSaveTest() throws Exception {
byte[] csvFile = unzip(zipFile);
String csvFileAsString = new String(csvFile, StandardCharsets.UTF_8);
List<String> actualCsvLines = Arrays.asList(csvFileAsString.split("\n"));
List<String> expectedCsvLines = new ArrayList<>(List.of("functionId,variableId,functionReference,value",

// Including "\uFEFF" indicates the UTF-8 BOM at the start.
List<String> expectedCsvLines = new ArrayList<>(List.of("\uFEFFfunctionId,variableId,functionReference,value",
"l1,GEN,2.9,500.1",
"l2,GEN,2.8,500.2"));

Expand Down
12 changes: 9 additions & 3 deletions ...java/org/gridsuite/sensitivityanalysis/server/service/SensitivityAnalysisServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,9 @@ public void testNoNKStillOK() throws Exception {
byte[] csv = unzip(zip);
String csvStr = new String(csv, StandardCharsets.UTF_8);
List<String> actualLines = Arrays.asList(csvStr.split("\n"));
List<String> expectedLines = new ArrayList<>(List.of("functionId,variableId,functionReference,value",

// Including "\uFEFF" indicates the UTF-8 BOM at the start.
List<String> expectedLines = new ArrayList<>(List.of("\uFEFFfunctionId,variableId,functionReference,value",
"l1,GEN,2.9,500.1",
"l2,GEN,2.8,500.2",
"l3,LOAD,2.1,500.9"));
Expand All @@ -357,7 +359,9 @@ public void testNoNKStillOK() throws Exception {
byte[] csv2 = unzip(zip2);
String csvStr2 = new String(csv2, StandardCharsets.UTF_8);
List<String> actualLines2 = Arrays.asList(csvStr2.split("\n"));
List<String> expectedLines2 = new ArrayList<>(List.of("functionId,variableId,contingencyId,functionReference,value,functionReferenceAfter,valueAfter"));

// Including "\uFEFF" indicates the UTF-8 BOM at the start.
List<String> expectedLines2 = new ArrayList<>(List.of("\uFEFFfunctionId,variableId,contingencyId,functionReference,value,functionReferenceAfter,valueAfter"));

actualLines2.sort(String::compareTo);
expectedLines2.sort(String::compareTo);
Expand Down Expand Up @@ -446,7 +450,9 @@ private void testNoN(boolean specific) throws Exception {
byte[] csv = unzip(zip);
String csvStr = new String(csv, StandardCharsets.UTF_8);
List<String> actualLines = Arrays.asList(csvStr.split("\n"));
List<String> expectedLines = new ArrayList<>(List.of("functionId,variableId,contingencyId,functionReference,value,functionReferenceAfter,valueAfter",

// Including "\uFEFF" indicates the UTF-8 BOM at the start.
List<String> expectedLines = new ArrayList<>(List.of("\uFEFFfunctionId,variableId,contingencyId,functionReference,value,functionReferenceAfter,valueAfter",
"l1,GEN,a1,0.0,0.0,2.7,500.3",
"l1,GEN,a2,0.0,0.0,2.6,500.4",
"l1,GEN,a3,0.0,0.0,2.5,500.5",
Expand Down
Loading