Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update powsybl-core dependency #70

Merged
merged 24 commits into from
Apr 4, 2024
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions .github/workflows/maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on:
branches:
- 'main'
- 'release-v**'
- 'full-sonar-analysis-**'
pull_request:

jobs:
Expand All @@ -17,12 +18,13 @@ jobs:

steps:
- name: Checkout sources
uses: actions/checkout@v1
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- name: Set up JDK 11
uses: actions/setup-java@v1
- name: Set up JDK 17
uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0
with:
java-version: 11
distribution: 'temurin'
java-version: '17' # arquillian tests fail on 11.0.15+, should probably upgrade there instead

- name: Build with Maven
run: mvn --batch-mode package
Expand Down
10 changes: 3 additions & 7 deletions computation-mpi/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<parent>
<groupId>com.powsybl</groupId>
<artifactId>powsybl-hpc</artifactId>
<version>2.12.0-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
</parent>

<artifactId>powsybl-computation-mpi</artifactId>
Expand Down Expand Up @@ -56,10 +56,6 @@
<groupId>com.powsybl</groupId>
<artifactId>powsybl-tools</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>

<!-- Test dependencies -->
<dependency>
Expand All @@ -68,8 +64,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,16 @@
import com.google.common.base.Splitter;
import com.google.common.base.Splitter.MapSplitter;
import com.powsybl.commons.PowsyblException;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.time.ZonedDateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;

Expand Down Expand Up @@ -72,7 +73,7 @@ private static final class TaskExecution {
final int jobId;
final int taskIndex;

final DateTime startTime;
final ZonedDateTime startTime;
final int slaveRank;
final int slaveThread;
final long inputMessageSize;
Expand All @@ -83,7 +84,7 @@ private static final class TaskExecution {
Long workingDataSize;
Integer exitCode;

private TaskExecution(int taskId, int jobId, int taskIndex, DateTime startTime, int slaveRank, int slaveThread, long inputMessageSize) {
private TaskExecution(int taskId, int jobId, int taskIndex, ZonedDateTime startTime, int slaveRank, int slaveThread, long inputMessageSize) {
this.taskId = taskId;
this.jobId = jobId;
this.taskIndex = taskIndex;
Expand Down Expand Up @@ -177,7 +178,7 @@ private void read(Handler handler) throws IOException {
taskId = Integer.parseInt(tokens[1]);
jobId = Integer.parseInt(tokens[2]);
int taskIndex = Integer.parseInt(tokens[3]);
DateTime startTime = DateTime.parse(tokens[4]);
ZonedDateTime startTime = ZonedDateTime.parse(tokens[4]);
int slaveRank = Integer.parseInt(tokens[5]);
int slaveThread = Integer.parseInt(tokens[6]);
long inputMessageSize = Long.parseLong(tokens[7]);
Expand Down Expand Up @@ -297,7 +298,7 @@ public void logJobEnd(int jobId) {
}

@Override
public void logTaskStart(int taskId, int jobId, int taskIndex, DateTime startTime, int slaveRank, int slaveThread, long inputMessageSize) {
public void logTaskStart(int taskId, int jobId, int taskIndex, ZonedDateTime startTime, int slaveRank, int slaveThread, long inputMessageSize) {
try {
internalWriter.write(TASK_START_KEY);
internalWriter.write(CSV_SEPARATOR);
Expand Down Expand Up @@ -386,7 +387,7 @@ public void onTaskEnd(StatisticsReader.TaskExecution task, StatisticsReader.JobE
task.taskIndex + CSV_SEPARATOR +
job.commandId + CSV_SEPARATOR +
(job.tags != null ? mapJoiner.join(job.tags) : "") + CSV_SEPARATOR +
task.startTime.toString("dd/MM/YYYY HH:mm:ss") + CSV_SEPARATOR +
task.startTime.format(DateTimeFormatter.ofPattern("dd/MM/yyyy HH:mm:ss")) + CSV_SEPARATOR +
task.slaveRank + CSV_SEPARATOR +
task.slaveThread + CSV_SEPARATOR +
task.inputMessageSize + CSV_SEPARATOR +
Expand Down Expand Up @@ -498,8 +499,8 @@ public static void exportBusyCores(Path dbDir, String dbName) throws IOException
Path busyCoresCsv = dbDir.resolve("busy-cores.csv");
logWritingPath(busyCoresCsv);

final DateTime[] min = new DateTime[1];
final DateTime[] max = new DateTime[1];
final ZonedDateTime[] min = new ZonedDateTime[1];
final ZonedDateTime[] max = new ZonedDateTime[1];
try (StatisticsReader reader = new StatisticsReader(csv)) {
reader.read(new StatisticsReader.AbstractHandler() {

Expand All @@ -509,7 +510,7 @@ public void onTaskEnd(StatisticsReader.TaskExecution task, StatisticsReader.JobE
min[0] = task.startTime;
}
if (task.taskDuration != null) {
DateTime endTime = task.startTime.plusMillis((int) (long) task.taskDuration);
ZonedDateTime endTime = task.startTime.plusNanos((int) (long) (task.taskDuration * 1e6));
if (max[0] == null || endTime.compareTo(max[0]) > 0) {
max[0] = endTime;
}
Expand All @@ -519,15 +520,15 @@ public void onTaskEnd(StatisticsReader.TaskExecution task, StatisticsReader.JobE
});
}

final int secs = (int) new Duration(min[0], max[0]).getStandardSeconds() + 1;
final int secs = (int) Duration.between(min[0], max[0]).getSeconds() + 1;
final int[] busyCores = new int[secs];
Arrays.fill(busyCores, 0);
try (StatisticsReader reader = new StatisticsReader(csv)) {
reader.read(new StatisticsReader.AbstractHandler() {

@Override
public void onTaskEnd(StatisticsReader.TaskExecution task, StatisticsReader.JobExecution job) {
int s1 = (int) new Duration(min[0], task.startTime).getStandardSeconds();
int s1 = (int) Duration.between(min[0], task.startTime).getSeconds();
int s2;
if (task.taskDuration != null) {
s2 = s1 + (int) ((float) task.taskDuration / 1000);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public MpiComputationManager(Path localDir, MpiNativeServices nativeServices, Mp
Path statisticsDbDir, String statisticsDbName, MpiExecutorContext executorContext,
int coresPerRank, boolean verbose, Path stdOutArchive) throws IOException, InterruptedException {
this(localDir, new MpiJobSchedulerImpl(nativeServices, statisticsFactory, statisticsDbDir, statisticsDbName, coresPerRank,
verbose, executorContext.getSchedulerExecutor(), stdOutArchive), executorContext);
verbose, executorContext.getSchedulerExecutor(), stdOutArchive), executorContext);
}

public MpiComputationManager(Path localDir, MpiJobScheduler scheduler, MpiExecutorContext executorContext) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,31 @@
*/
package com.powsybl.computation.mpi;

import com.google.common.collect.ImmutableMap;
import com.powsybl.computation.ComputationResourcesStatus;

import java.time.ZonedDateTime;
import java.util.Map;
import org.joda.time.DateTime;

/**
*
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
*/
class MpiComputationResourcesStatus implements ComputationResourcesStatus {

private final DateTime date;
private final ZonedDateTime date;

private int availableCores = 0;

private int busyCores = 0;

MpiComputationResourcesStatus(MpiResources resources) {
date = new DateTime();
date = ZonedDateTime.now();
availableCores = resources.getAvailableCores();
busyCores = resources.getBusyCores();
}

@Override
public DateTime getDate() {
public ZonedDateTime getDate() {
return date;
}

Expand All @@ -46,7 +46,7 @@ public int getBusyCores() {

@Override
public Map<String, Integer> getBusyCoresPerApp() {
return ImmutableMap.of("all", busyCores);
return Map.of("all", busyCores);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,10 @@
*/
package com.powsybl.computation.mpi;

import com.google.common.collect.ImmutableMap;
import com.google.protobuf.ByteString;
import com.powsybl.commons.PowsyblException;
import com.powsybl.computation.*;
import com.powsybl.computation.mpi.generated.Messages;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -21,6 +18,8 @@
import java.io.InputStream;
import java.net.URI;
import java.nio.file.*;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
Expand All @@ -38,7 +37,7 @@ class MpiJobSchedulerImpl implements MpiJobScheduler {

private static final int TIMEOUT = 10; // ms

private static final Map<String, String> ZIP_FS_ENV = ImmutableMap.of("create", "true");
private static final Map<String, String> ZIP_FS_ENV = Map.of("create", "true");

private final MpiNativeServices nativeServices;

Expand Down Expand Up @@ -186,10 +185,10 @@ private static Messages.Task.OutputFile.PostProcessor createPostProcessor(FilePo

private static Messages.Task.Environment createEnvMessage(Map<String, String> variables) {
Messages.Task.Environment.Builder builder = Messages.Task.Environment.newBuilder();
for (Map.Entry<String, String> var : variables.entrySet()) {
for (Map.Entry<String, String> entry : variables.entrySet()) {
builder.addVariable(Messages.Task.Variable.newBuilder()
.setName(var.getKey())
.setValue(var.getValue())
.setName(entry.getKey())
.setValue(entry.getValue())
.build());
}
return builder.build();
Expand Down Expand Up @@ -227,7 +226,7 @@ private Messages.Task createTaskMessage(MpiJob job, MpiRank rank, Command comman
.setInitJob(initJob);

//
// select which files have to be send with the message
// select which files have to be sent with the message
//
for (InputFile file : command.getInputFiles()) {
if (file.dependsOnExecutionNumber()) {
Expand Down Expand Up @@ -365,7 +364,7 @@ private boolean processJobs(List<MpiTask> completedTasks) throws IOException {
checkTaskCompletionTime += System.currentTimeMillis() - t0;
}
if (!completedTasks.isEmpty()) {
DateTime endTime = DateTime.now();
ZonedDateTime endTime = ZonedDateTime.now();

// release cores as fast as possible
completedTasks.forEach(task -> {
Expand Down Expand Up @@ -418,7 +417,7 @@ private boolean startTasks(MpiJob job) throws IOException {
command.toString(-1), allocatedCores, job.getWorkingDir());
}

DateTime startTime = DateTime.now();
ZonedDateTime startTime = ZonedDateTime.now();

// encode task messages
int oldTaskIndex = taskIndex;
Expand Down Expand Up @@ -489,7 +488,7 @@ private void processCompletedTasks(MpiJob job, List<MpiTask> completedTasks) thr

for (MpiTask task : completedTasks) {
// duration of the task seen by the master in ms
long taskDurationSeenByMaster = new Duration(task.getStartTime(), task.getEndTime()).getMillis();
long taskDurationSeenByMaster = Duration.between(task.getStartTime(), task.getEndTime()).getNano() / 1000000;

// decode task result messages
Messages.TaskResult message = Messages.TaskResult.parseFrom(task.getResultMessage());
Expand All @@ -504,7 +503,7 @@ private void processCompletedTasks(MpiJob job, List<MpiTask> completedTasks) thr
copyFileToWorkingDirectory(outputFile, job, stdOutGzFileName);

// archive standard output of problematic tasks
if ((stdOutArchive != null) && (message.getExitCode() != 0) && outputFile.getName().equals(stdOutGzFileName)) {
if (stdOutArchive != null && message.getExitCode() != 0 && outputFile.getName().equals(stdOutGzFileName)) {
try (FileSystem archiveFileSystem = FileSystems.newFileSystem(URI.create("jar:file:" + stdOutArchive.toUri().getPath()), ZIP_FS_ENV)) {
Path dir = archiveFileSystem.getPath("/").resolve("job-" + job.getId());
Files.createDirectories(dir);
Expand All @@ -528,7 +527,7 @@ private void processCompletedTasks(MpiJob job, List<MpiTask> completedTasks) thr
LOGGER.error(e.toString(), e);
}

// update execution statistics
// update execution statistics - durations are in ms
statistics.logTaskEnd(task.getId(),
taskDurationSeenByMaster,
message.getCommandDurationList(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import java.io.Writer;
import java.util.List;
import java.util.Map;
import org.joda.time.DateTime;
import java.time.ZonedDateTime;

/**
*
Expand Down Expand Up @@ -54,7 +54,7 @@ public interface MpiStatistics extends AutoCloseable {
* @param slaveThread slave thread that has managed the task
* @param inputMessageSize input message size in bytes
*/
void logTaskStart(int taskId, int jobId, int taskIndex, DateTime startTime, int slaveRank, int slaveThread, long inputMessageSize);
void logTaskStart(int taskId, int jobId, int taskIndex, ZonedDateTime startTime, int slaveRank, int slaveThread, long inputMessageSize);

/**
* Log a task end.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/
package com.powsybl.computation.mpi;

import org.joda.time.DateTime;
import java.time.ZonedDateTime;

/**
*
Expand All @@ -22,13 +22,13 @@ class MpiTask {

private final byte[] message;

private final DateTime startTime;
private final ZonedDateTime startTime;

private DateTime endTime;
private ZonedDateTime endTime;

private byte[] resultMessage;

MpiTask(int id, Core core, int index, byte[] message, DateTime startTime) {
MpiTask(int id, Core core, int index, byte[] message, ZonedDateTime startTime) {
this.id = id;
this.core = core;
this.index = index;
Expand Down Expand Up @@ -60,15 +60,15 @@ byte[] getMessage() {
return message;
}

DateTime getStartTime() {
ZonedDateTime getStartTime() {
return startTime;
}

DateTime getEndTime() {
ZonedDateTime getEndTime() {
return endTime;
}

void setEndTime(DateTime endTime) {
void setEndTime(ZonedDateTime endTime) {
this.endTime = endTime;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import java.io.Writer;
import java.util.List;
import java.util.Map;
import org.joda.time.DateTime;
import java.time.ZonedDateTime;

/**
*
Expand All @@ -33,7 +33,7 @@ public void logJobEnd(int jobId) {
}

@Override
public void logTaskStart(int taskId, int jobId, int taskIndex, DateTime startTime, int slaveRank, int slaveThread, long inputMessageSize) {
public void logTaskStart(int taskId, int jobId, int taskIndex, ZonedDateTime startTime, int slaveRank, int slaveThread, long inputMessageSize) {
// no-op implementation
}

Expand Down
Loading