Skip to content

Commit

Permalink
Merge pull request #23 from cnescatlab/feature/add_external_tool_anal…
Browse files Browse the repository at this point in the history
…ysis

Add external tool analysis
  • Loading branch information
Topin2001 authored Jul 23, 2024
2 parents 4510233 + 6b4bf48 commit cbad5fe
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 93 deletions.
10 changes: 6 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>fr.cnes.sonar.plugins.scan</groupId>
Expand All @@ -18,7 +20,7 @@
<gson.version>2.11.0</gson.version>
<junit.jupiter.api.version>5.11.0-M2</junit.jupiter.api.version>
<jacoco-maven-plugin.version>0.8.12</jacoco-maven-plugin.version>
<jdk.version>17</jdk.version>
<jdk.version>17</jdk.version>
<pluginUrl>https://cnes.fr</pluginUrl>
<pluginOrganizationName>CNES</pluginOrganizationName>
</properties>
Expand Down Expand Up @@ -48,7 +50,7 @@
</developer>
<developer>
<id>Topin2001</id>
<name>Topin</name>
<name>Topin</name>
</developer>
</developers>

Expand Down Expand Up @@ -173,4 +175,4 @@
</plugins>
</build>

</project>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,6 @@ public abstract class AbstractTask implements RequestHandler {
* @throws InterruptedException when a command is not finished
*/
protected String executeCommand(final String command) throws IOException, InterruptedException {
// log the command to execute
LOGGER.info(command);

// prepare a string builder for the output gathering
final StringBuilder output = new StringBuilder();
Expand Down
147 changes: 109 additions & 38 deletions src/main/java/fr/cnes/sonar/plugins/scan/tasks/AnalysisTask.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,41 +17,48 @@
package fr.cnes.sonar.plugins.scan.tasks;

import fr.cnes.sonar.plugins.scan.utils.StringManager;

import org.sonar.api.config.Configuration;
import org.sonar.api.server.ws.Request;
import org.sonar.api.server.ws.Response;
import org.sonar.api.utils.text.JsonWriter;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Type;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.*;

/**
* Execute the scan of a project
*
* @author lequal
*/
public class AnalysisTask extends AbstractTask {

private final Configuration config;

public AnalysisTask(Configuration config){
public AnalysisTask(Configuration config) {
this.config = config;
}

/**
* Logged message when a file can not be deleted
*/
private static final String FILE_DELETION_ERROR =
"The following file could not be deleted: %s.";
private static final String FILE_DELETION_ERROR = "The following file could not be deleted: %s.";
/**
* Logged message when a file can not be set as executable
*/
private static final String FILE_PERMISSIONS_ERROR =
"Permissions of the following file could not be changed: %s.";
private static final String FILE_PERMISSIONS_ERROR = "Permissions of the following file could not be changed: %s.";
/**
* Just a slash
*/
Expand All @@ -69,55 +76,67 @@ public AnalysisTask(Configuration config){
*/
private static final String NEW_LINE = "\n";



/**
* Execute the scan of a project
* @param projectName name of the project to analyze
* @param projectFolder url of the folder containing the project to analyze
*
* @param projectName name of the project to analyze
* @param projectFolder url of the folder containing the project to
* analyze
* @param sonarProjectProperties the sonar-project.properties as string
* @param qualityProfiles the quality profiles as string
* @return logs
* @throws IOException when a file writing goes wrong
* @throws IOException when a file writing goes wrong
* @throws InterruptedException when a command is not finished
*/
private String analyze(final String projectName, final String projectFolder, final String sonarProjectProperties)
private String analyze(final String projectName, final String projectFolder, final String sonarProjectProperties,
final String qualityProfiles)
throws IOException, InterruptedException {
// setting a timer based on user's timeout property configuration
final Integer timeout = Integer.parseInt(config.get(StringManager.string(StringManager.TIMEOUT_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)));
final Integer timeout = Integer.parseInt(config.get(StringManager.string(StringManager.TIMEOUT_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)));
final ExecutorService service = Executors.newSingleThreadExecutor();
try {
final Runnable task = () -> {
try {
// path where spp should be written
final String sppPath = String.format(StringManager.string(StringManager.CNES_SPP_PATH),
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)), projectFolder);
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)),
projectFolder);

// write sonar-project.properties in the project folder
writeTextFile(sppPath, sonarProjectProperties);
// build the scan command
final String analysisCommand = String.format(
StringManager.string(StringManager.CNES_COMMAND_SCAN),
config.get(StringManager.string(StringManager.SCANNER_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)),
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)),
config.get(StringManager.string(StringManager.SCANNER_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)),
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)),
projectFolder,
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)),
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)),
projectFolder);
final File script = createScript(projectFolder, analysisCommand);
final File script = createScript(projectFolder, analysisCommand, qualityProfiles);

// string formatted date as string
final String date = new SimpleDateFormat(
StringManager.string(StringManager.DATE_PATTERN)).format(new Date());

// export log file
final String logPath = String.format(StringManager.string(StringManager.CNES_LOG_PATH),
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)), date, projectName);
config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)),
date, projectName);

// scan execution
final String scriptCommand = config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)) +
final String scriptCommand = config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)) +
SLASH + projectFolder + SLASH + CAT_SCAN_SCRIPT;
log(executeCommand(scriptCommand));
// log output file
final String path = config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING)) +
final String path = config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)) +
SLASH + projectFolder + SLASH + CAT_LOG_FILE;
for (final String line : Files.readAllLines(Paths.get(path))) {
log(line + NEW_LINE);
Expand All @@ -137,7 +156,6 @@ private String analyze(final String projectName, final String projectFolder, fin
LOGGER.severe(e.getMessage());
}


};
final Future<?> execution = service.submit(task);

Expand All @@ -153,13 +171,13 @@ private String analyze(final String projectName, final String projectFolder, fin
service.shutdown();
}


// return the complete logs
return getLogs();
}

/**
* Export the sonar-project.properties in the corresponding folder
*
* @param path Output folder
* @param data Data to write
* @throws IOException when a file writing goes wrong
Expand All @@ -170,17 +188,18 @@ private void writeTextFile(final String path, final String data) throws IOExcept

// create the writer
// true to append; false to overwrite.
try(FileWriter fileWriter = new FileWriter(spp, false)) {
try (FileWriter fileWriter = new FileWriter(spp, false)) {
// write the data
fileWriter.write(data);
}
}

/**
* Use the user's request to launch an scan
* @param request request coming from the user
*
* @param request request coming from the user
* @param response response to send to the user
* @throws IOException when communicating with the client
* @throws IOException when communicating with the client
* @throws InterruptedException ...
*/
@Override
Expand All @@ -196,9 +215,11 @@ public void handle(final Request request, final Response response)
StringManager.string(StringManager.ANALYZE_FOLDER_NAME));
final String sonarProjectProperties = request.mandatoryParam(
StringManager.string(StringManager.ANALYZE_SPP_NAME));
final String qualityProfiles = request.mandatoryParam(
StringManager.string(StringManager.ANALYZE_QUALITY_PROFILES_NAME));

// concrete scan
final String result = analyze(projectName, workspace, sonarProjectProperties);
final String result = analyze(projectName, workspace, sonarProjectProperties, qualityProfiles);

// write the json response
try (JsonWriter jsonWriter = response.newJsonWriter()) {
Expand All @@ -209,33 +230,83 @@ public void handle(final Request request, final Response response)
}
}

private String setupExternalTools(String qualityProfile) {
StringBuilder setupExternalTools = new StringBuilder();

Gson gson = new Gson();
Type outerListType = new TypeToken<List<String>>() {
}.getType();
List<String> outerList = gson.fromJson(qualityProfile, outerListType);
List<List<String>> qualityProfiles = new ArrayList<>();
for (String innerJson : outerList) {
Type innerListType = new TypeToken<List<String>>() {
}.getType();
List<String> innerList = gson.fromJson(innerJson, innerListType);
qualityProfiles.add(innerList);
}

for (List<String> qp : qualityProfiles) {
if (qp.get(0).equals("py")) {
LOGGER.info("Setup pylint");
// Detect and run correct pylintrc according to RNC
String pylintrc = "/opt/python/pylintrc_RNC2015_D";
switch (qp.get(1)) {
case "RNC A":
pylintrc = "/opt/python/pylintrc_RNC2015_A_B";
break;
case "RNC B":
pylintrc = "/opt/python/pylintrc_RNC2015_A_B";
break;
case "RNC C":
pylintrc = "/opt/python/pylintrc_RNC2015_C";
break;
default:
break;
}
setupExternalTools.append(
"\npylint --rcfile=" + pylintrc
+ " --load-plugins=pylint_sonarjson --output-format=sonarjson --output=pylint-report.json *.py");
}
if (qp.contains("docker")) {
LOGGER.info("Setup hadolint");
setupExternalTools.append(
"\nhadolint -f sonarqube --no-fail --config=/opt/hadolint/hadolint_RNC_A_B_C_D.yaml Dockerfile > hadolint-report.json");
}
}
return setupExternalTools.toString();
}

/**
* Create a temporary script containing dedicated command executing sonar-scanner
* @param project repository containing the source code
* Create a temporary script containing dedicated command executing
* sonar-scanner
*
* @param project repository containing the source code
* @param commandLine command line to execute
* @return The created file
*/
private File createScript(final String project, final String commandLine) {
private File createScript(final String project, final String commandLine, final String qualityProfiles) {
// path to the workspace
final String workspace = config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY)).orElse(StringManager.string(StringManager.DEFAULT_STRING))+
SLASH +project+ SLASH;
final String workspace = config.get(StringManager.string(StringManager.WORKSPACE_PROP_DEF_KEY))
.orElse(StringManager.string(StringManager.DEFAULT_STRING)) +
SLASH + project + SLASH;
// create script in a file located in the project's repository
final File scriptOutput = new File(workspace + CAT_SCAN_SCRIPT);

String setupExternalTools = setupExternalTools(qualityProfiles);

// Write all command lines in a single temporary script
try (
FileWriter script = new FileWriter(scriptOutput)
){
script.write("#!/bin/bash -e");
script.write("\ncd "+workspace);
script.write(StringManager.string(StringManager.CNES_LOG_SEPARATOR)+commandLine);
LOGGER.info("commandLine : " + commandLine);
FileWriter script = new FileWriter(scriptOutput)) {
script.write("#!/bin/bash");
script.write("\ncd " + workspace);
script.write("\n" + setupExternalTools);
script.write(StringManager.string(StringManager.CNES_LOG_SEPARATOR) + commandLine);
} catch (IOException e) {
LOGGER.severe(e.getMessage());
}

// give execution rights on the script
if(!scriptOutput.setExecutable(true)) {
if (!scriptOutput.setExecutable(true)) {
LOGGER.severe(String.format(FILE_PERMISSIONS_ERROR, scriptOutput.getName()));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ public final class StringManager {
* Property for action 1 (scan) param 6 description
*/
public static final String ANALYZE_SPP_DESC = "cnes.action.analyze.param.spp.desc";
/**
* Property for action 1 (scan) param 7 description
*/
public static final String ANALYZE_QUALITY_PROFILES_DESC =
"cnes.action.analyze.param.profiles.desc";
/**
* Property for quality profiles separator
*/
Expand Down Expand Up @@ -191,6 +196,10 @@ public final class StringManager {
* Define the name of the projects's sonar-project.properties parameter
*/
public static final String ANALYZE_SPP_NAME = "cnes.action.analyze.param.spp.name";
/**
* Define the name of the project's quality profiles parameter
*/
public static final String ANALYZE_QUALITY_PROFILES_NAME = "cnes.action.analyze.param.profiles.name";
/**
* Define the name of the returned log filed
*/
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/fr/cnes/sonar/plugins/scan/ws/CnesWs.java
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@ private void analyzeAction(final NewController controller) {
newParam = analysis.createParam(StringManager.string(StringManager.ANALYZE_SPP_NAME));
newParam.setDescription(StringManager.string(StringManager.ANALYZE_SPP_DESC));
newParam.setRequired(true);
// quality profiles parameter
newParam = analysis.createParam(StringManager.string(StringManager.ANALYZE_QUALITY_PROFILES_NAME));
newParam.setDescription(StringManager.string(StringManager.ANALYZE_QUALITY_PROFILES_DESC));
newParam.setRequired(true);
}


Expand Down
Loading

0 comments on commit cbad5fe

Please sign in to comment.