Skip to content

Commit

Permalink
Use the officially supported apache mina sshd in m2k-func
Browse files Browse the repository at this point in the history
The various git command we use in m2k func now uses the officially
supported ssh implementation for the transport.

Added tests to see the transport is working well with ed25519 keys

Notice - the StrictHost option is now set using a standard ssh_config
(see man ssh_config) file which is assumed to be where the key is located -
so now we can just drop the id_rsa file in a well known location along
with the rest of the .ssh files:

/etc/.ssh/config
/etc/.ssh/known_hosts
/etc/.ssh/id_rsa

Signed-off-by: Roy Golan <[email protected]>
  • Loading branch information
rgolangh committed Jun 24, 2024
1 parent bfc6da8 commit 4b1aac7
Show file tree
Hide file tree
Showing 13 changed files with 209 additions and 59 deletions.
11 changes: 6 additions & 5 deletions .github/workflows/m2k-func.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ name: Build and push move2kube Knative function container image
on:
workflow_call:
inputs:
push_pr:
it_mode:
required: false
type: boolean
default: true
default: false
workflow_dispatch:
push:
branches: [ "main" ]
Expand Down Expand Up @@ -49,20 +49,20 @@ jobs:
${{ env.WORKDIR }}/src/main/docker/Dockerfile.jvm
- name: Buildah push to OCI Arcive
if: ${{ ! inputs.push_pr }}
if: ${{ inputs.it_mode }}
run: |
buildah push serverless-workflow-m2k-kfunc:${{ github.sha }} \
oci-archive:serverless-workflow-m2k-kfunc-${{ github.sha }}.tar:kind.local/orchestrator/serverless-workflow-m2k-kfunc:${{ github.sha }}
- name: Save OCI archive
if: ${{ ! inputs.push_pr }}
if: ${{ inputs.it_mode }}
uses: actions/upload-artifact@v4
with:
name: serverless-workflow-m2k-kfunc-${{ github.sha }}.tar
path: serverless-workflow-m2k-kfunc-${{ github.sha }}.tar

- name: Push To quay.io
if: ${{ inputs.push_pr }}
if: ${{ ! inputs.it_mode }}
id: push-to-quay
uses: redhat-actions/push-to-registry@v2
with:
Expand All @@ -76,6 +76,7 @@ jobs:
run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"

- uses: actions/github-script@v7
if: ${{ inputs.it_mode }}
id: get_pr_data
with:
script: |
Expand Down
11 changes: 2 additions & 9 deletions .github/workflows/move2kube-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
uses: ./.github/workflows/m2k-func.yaml
secrets: inherit
with:
push_pr: false
it_mode: true

run-m2k-e2e:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -98,10 +98,6 @@ jobs:
--type merge \
-p '{"data":{"kubernetes.podspec-init-containers": "enabled", "kubernetes.podspec-securitycontext": "enabled"}}'
###### workaround till https://issues.redhat.com/browse/FLPATH-892 is solved
# yq --inplace '.spec.podTemplate.container |= ( . + {"imagePullPolicy": "IfNotPresent"} )' manifests/01-sonataflow_m2k.yaml
###### end workaround
yq --inplace '.spec.podTemplate.container |= ( . + {"env": [{"name": "K_SINK", "value": "http://broker-ingress.knative-eventing.svc.cluster.local/sonataflow-infra/default"}]} )' manifests/01-sonataflow_m2k.yaml
# Disable persistence for e2e tests
Expand All @@ -128,11 +124,8 @@ jobs:
- name: Deploy Knative function
run: |
###### workaround till https://issues.redhat.com/browse/FLPATH-892 is solved
yq --inplace '.spec.template.spec.containers[0] |= ( . + {"imagePullPolicy": "IfNotPresent"} )' e2e/resources/knative-service.yaml
###### end workaround
kubectl apply -f e2e/resources/move2kube-configmaps.yaml
yq --inplace '.spec.template.spec.containers[0] |= ( . + {"image": "kind.local/orchestrator/serverless-workflow-m2k-kfunc:${{ github.sha }}"} )' e2e/resources/knative-service.yaml
# deploy the manifests created by the ${{ steps.build-image.outputs.image }}"
kubectl apply -f e2e/resources/knative-service.yaml
kubectl apply -f e2e/resources/knative-resources.yaml
kubectl wait ksvc m2k-save-transformation-func --for=condition=Ready=true --timeout=5m
Expand Down
10 changes: 10 additions & 0 deletions .gitleaks.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[allowlist]
description = "Global Allowlist"

# Ignore based on any subset of the file path
paths = [

# Ignore some long path
'''move2kube\/m2k-func\/src\/test\/resources\/m2k-test-ssh-id_ed25519$''',
]

9 changes: 0 additions & 9 deletions e2e/resources/knative-service.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,3 @@
apiVersion: v1
data:
config: |
Host *
StrictHostKeyChecking no
kind: ConfigMap
metadata:
name: m2k-ssh-config
---
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
Expand Down
8 changes: 8 additions & 0 deletions e2e/resources/move2kube-configmaps.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: m2k-ssh-config
data:
config: |
Host *
StrictHostKeyChecking no
19 changes: 7 additions & 12 deletions move2kube/m2k-func/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,8 @@
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.ssh.jsch</artifactId>
<artifactId>org.eclipse.jgit.ssh.apache</artifactId>
<version>${jgit-version}</version>
<exclusions>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.mwiede</groupId>
<artifactId>jsch</artifactId>
<version>0.2.9</version>
</dependency>
<dependency>
<groupId>dev.parodos</groupId>
Expand Down Expand Up @@ -125,6 +114,12 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-rest-client-reactive-jackson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-git</artifactId>
<version>2.10.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<profile>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package dev.parodos.service;

import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;

import jakarta.enterprise.context.ApplicationScoped;

import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.CommitCommand;
import org.eclipse.jgit.api.Git;
Expand All @@ -14,16 +18,11 @@
import org.eclipse.jgit.api.errors.InvalidRemoteException;
import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.Transport;
import org.eclipse.jgit.transport.ssh.jsch.JschConfigSessionFactory;
import org.eclipse.jgit.transport.ssh.jsch.OpenSshConfig;
import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.eclipse.microprofile.config.ConfigProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.nio.file.Path;

@ApplicationScoped
public class GitServiceImpl implements GitService {
private static final Logger log = LoggerFactory.getLogger(GitServiceImpl.class);
Expand Down Expand Up @@ -97,27 +96,18 @@ public static TransportConfigCallback getTransport(Path sshKeyPath) throws IOExc
throw new IOException("SSH key file at '%s' does not exists".formatted(sshKeyPath.toString()));
}

var sshSessionFactory = new JschConfigSessionFactory() {
@Override
protected void configure(OpenSshConfig.Host host, Session session) {
session.setConfig("StrictHostKeyChecking", "no");
session.setConfig("PreferredAuthentications", "publickey");
}
var sshSessionFactory = new SshdSessionFactoryBuilder()
.setDefaultIdentities(f -> Collections.singletonList(sshKeyPath))
.setPreferredAuthentications("publickey")
.setSshDirectory(sshKeyPath.getParent().toFile())
.setHomeDirectory(sshKeyPath.getParent().toFile())
.build(null);

@Override
protected JSch createDefaultJSch(FS fs) throws JSchException {
JSch defaultJSch = super.createDefaultJSch(fs);
defaultJSch.removeAllIdentity();
defaultJSch.addIdentity(sshKeyPath.toString());
return defaultJSch;
}
};
return new TransportConfigCallback() {
@Override
public void configure(Transport transport) {
SshTransport sshTransport = (SshTransport) transport;
sshTransport.setSshSessionFactory(sshSessionFactory);

}
};
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package dev.parodos.service;

import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Collections;
import java.util.Comparator;

import org.apache.sshd.common.keyprovider.ClassLoadableResourceKeyPairProvider;
import org.apache.sshd.git.GitLocationResolver;
import org.apache.sshd.git.pack.GitPackCommandFactory;
import org.apache.sshd.server.SshServer;
import org.apache.sshd.server.auth.password.AcceptAllPasswordAuthenticator;
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
import org.apache.sshd.sftp.server.SftpSubsystemFactory;
import org.eclipse.jgit.api.AddCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;


class GitServiceImplTest {
static SshServer sshd;
static Path serverDir;
static Path cloneDir;
static final String testFile = "test-file";
static final String testFileContent = "test-content";
GitServiceImpl underTest = new GitServiceImpl();

@Test
void cloneRepo() throws IOException, GitAPIException {
try (Git git = underTest.cloneRepo("ssh://%s:%d%s".formatted(sshd.getHost(), sshd.getPort(), serverDir.toString()), "", cloneDir)) {
Path testfile = cloneDir.resolve(testFile);
Assertions.assertTrue(Files.exists(testfile), "cloned file doesn't exists");
Assertions.assertEquals(Files.readString(testfile), testFileContent);
}
}

@Test
void push() throws IOException, GitAPIException {
try (Git git = underTest.cloneRepo("ssh://%s:%d%s".formatted(sshd.getHost(), sshd.getPort(), serverDir.toString()), "", cloneDir)) {
Files.write(cloneDir.resolve("push-test-file"), "foo".getBytes());
underTest.commit(git, "push test commit", ".");
Assertions.assertDoesNotThrow(() -> underTest.push(git));
}
}

@BeforeAll
static void init() throws IOException, GitAPIException {
// create an ssh server
sshd = SshServer.setUpDefaultServer();
sshd.setHost("localhost");
sshd.setPort(30022);
URL hostkey = GitServiceImplTest.class.getClassLoader().getResource("m2k-test-ssh-id_ed25519.pub");
SimpleGeneratorHostKeyProvider keyPairProvider = new SimpleGeneratorHostKeyProvider(Paths.get(hostkey.getPath()));
sshd.setKeyPairProvider(keyPairProvider);
keyPairProvider.loadKeys(null);
sshd.setPasswordAuthenticator(AcceptAllPasswordAuthenticator.INSTANCE);

serverDir = Files.createTempDirectory(
"m2kfunc-test-git-repo-server",
PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwxrwx")));

sshd.setSubsystemFactories(Collections.singletonList(new SftpSubsystemFactory()));
sshd.setCommandFactory(new GitPackCommandFactory(GitLocationResolver.constantPath(Paths.get("/"))));
sshd.setKeyPairProvider(new ClassLoadableResourceKeyPairProvider("test.key"));
sshd.setPublickeyAuthenticator((username, key, session) -> true);
sshd.start();

// crete git repo on the ssh server
Repository repo = new FileRepositoryBuilder().setWorkTree(serverDir.toFile()).build();
repo.create(true);
Files.write(serverDir.resolve(testFile).toAbsolutePath(), testFileContent.getBytes());
new AddCommand(repo).addFilepattern(".").call();
Git git = new Git(repo);
git.add().addFilepattern(".").call();
git.commit().setAuthor("me", "me@me").setMessage("first commit from test init").call();
}

@AfterAll
static void afterAll() throws IOException {
try (var walk = Files.walk(serverDir)) {
walk.sorted(Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
}

@BeforeEach
void setup() throws IOException {
cloneDir = Files.createTempDirectory(
"m2kfunc-test-git-repo-clone",
PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwxrwx")));
}

@AfterEach
void teardown() throws IOException {
try (var walk = Files.walk(cloneDir)) {
walk.sorted(Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
}

}
2 changes: 2 additions & 0 deletions move2kube/m2k-func/src/test/resources/application.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
ssh-priv-key-path=${PWD}/src/test/resources/m2k-test-ssh-id_ed25519
quarkus.log.level=INFO
2 changes: 2 additions & 0 deletions move2kube/m2k-func/src/test/resources/config
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Host *
StrictHostKeyChecking no
7 changes: 7 additions & 0 deletions move2kube/m2k-func/src/test/resources/m2k-test-ssh-id_ed25519
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACDyPQxDOPtUCJI386z0a2G6b3uWDSTBvB2Q3m4EHY4zZQAAAJAgrbKCIK2y
ggAAAAtzc2gtZWQyNTUxOQAAACDyPQxDOPtUCJI386z0a2G6b3uWDSTBvB2Q3m4EHY4zZQ
AAAECWBqGUjVKfROUwn54KARnsnOX+Y8PAAhHWJG3tYZaoyfI9DEM4+1QIkjfzrPRrYbpv
e5YNJMG8HZDebgQdjjNlAAAADXJnb2xhbkBmZWRvcmE=
-----END OPENSSH PRIVATE KEY-----
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPI9DEM4+1QIkjfzrPRrYbpve5YNJMG8HZDebgQdjjNl rgolan@fedora
27 changes: 27 additions & 0 deletions move2kube/m2k-func/src/test/resources/test.key
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEogIBAAKCAQEAxuhZr4YA7l0S/yij3IXXf9jcLxgHdenjRGrF8tGbYp5n7IQb
msbXcfBeUcsmtrBx02TB45nQ5Nox2Lqch5OyKbVo8FzO5m5RGmmHPUokgJGwBtgH
SwxlX/dJ3X/T0Hk7s9X04t4wfbopWRPFIf+tUMp65XxgAdOpfM/yO4MjoRB3zL3X
wI2KDecpwg10SMLiLXSuHZrMnYDMIie3WSBXur5OHQrNfLDLIvZQxAvgO0O3MaKn
Pcq3XgtXonagYniwtUSi0VqUYUjDUTjwr5pc0v+UIJap610Mx8fiT1B63+YBUbQe
C+Ol82FMUKGKM8igDm9xRBD1LFVYg1YLrdBCGwIDAQABAoIBAB8f1oDXuCeUWteg
dVuZeeogdfvMh8ZUreJTztu7HtRksyBYX7VtbeL/WNL1tf4aSAVrG8fQltZoqioe
sUWpv9Q09dG+xAVct8YpQyc9Bc80fNXlUebVbruAh4dobC2P+t6eGS6y0+WojrXI
mS1Dw8wDkw17084VX80PAPl9AMM/+EzxJvf09nDBj1vn5JJm3TsPC7NhR0Fhe7pz
yNY1CFu1sK6ZcaQdg4ratyRZ0LxBAbPK3fALnqJQqIMAwK9QW1lLQDpavZHKXooe
UEj84kfmOJPT7wjOgh9CjFsjYOGPdkIDq0dGJ01X+mEQ4stdBToosPSl0NLdcItv
KGR5F7ECgYEA/ofmSuZ2mO0e2MyUrBYdBj7NR4vW6Kprsvg0MzmGkj2SqPeYD/D2
9AHRErkZ1Vns3m8OafpMSGRraMs3KxTCZBrCTRwvEVQBge9e+K3yiMzTpCznQ/I0
7v7Vufn13gevgZmza0PelLemCuzOpF7jUfRw0d75sAYD0mxSRKC4GqMCgYEAyA5C
uG9ykp+fJg9gp8m3N8ZOXLK+Dt7Eycght1k+l0NandMj/+9AU7AUZGKLEPHSUSOF
15b7juerMiAvNv5AoUnJkUFdimC7jUJGv9TKxTOEhUgscZAScyppUU5lUk4oExdK
O7ttbM/Ou9wEIRBJ0W3/pK29fWZ3yFIvfirJ6ikCgYAT44iiN6nyvyyW4j2HyN6R
u1yNB6dOXOq3fF+P1SHn0XnhTB+Mt1aEsJOms+IJ4tH4e5MTwuQtD/O4p5BzBFdA
PTsLjXU8FGVdwteX9Perqt2qyXt0urtaJX2L37VPmSgkp172tcHxuvv1hJWNEIEQ
yVn7fEHkeEPaMG6pQCnCowKBgGJIh0TfE9Wu79wd7+les1GGblciRTc/AET1uoK+
KH7dyzYAVg5Vty+mMM6Ejze65g2Qux+IgHvbmwKcRzXoQU471vgyucbS8TFb3zA9
VYT+Y1urcpI0KqxDqMwWDLcbyJpgdcrUsNSlXzZxx+GKhAmM1exMouxpm+1hWw3L
7bjJAoGAAep1Yg8/b0NRJtYQK6duFccrVzeXXSJ6nYcHgquVmg+VEQKwdaZBVDwv
u5QZXEhcC7dziq07Sh4FziGJXKwNzV86860w+MxkJcqrWSuQkprw/3UcODtE+RyN
j4b8grxg4ejd0KtCYzLy+ZpTqZZuotENpXhhcOH9VQRhWYXSSJg=
-----END RSA PRIVATE KEY-----

0 comments on commit 4b1aac7

Please sign in to comment.