Signature algorithm selection should match the behaviour of OpenSSH

src/main/java/com/jcraft/jsch/JSch.java

@@ -227,6 +227,8 @@ public class JSch {
         Util.getSystemProperty("jsch.enable_pubkey_auth_query", "yes"));
     config.put("try_additional_pubkey_algorithms",
         Util.getSystemProperty("jsch.try_additional_pubkey_algorithms", "yes"));
+    config.put("use_openssh_rsa_pubkey_order",
+        Util.getSystemProperty("jsch.use_openssh_rsa_pubkey_order", "yes"));
     config.put("enable_auth_none", Util.getSystemProperty("jsch.enable_auth_none", "yes"));
     config.put("use_sftp_write_flush_workaround",
         Util.getSystemProperty("jsch.use_sftp_write_flush_workaround", "yes"));

src/main/java/com/jcraft/jsch/Session.java

@@ -3067,6 +3067,7 @@ private void applyConfig() throws JSchException {
     checkConfig(config, "server_host_key");
     checkConfig(config, "prefer_known_host_key_types");
     checkConfig(config, "enable_pubkey_auth_query");
+    checkConfig(config, "use_openssh_rsa_pubkey_order");
     checkConfig(config, "try_additional_pubkey_algorithms");
     checkConfig(config, "enable_auth_none");
     checkConfig(config, "use_sftp_write_flush_workaround");

src/main/java/com/jcraft/jsch/UserAuthPublicKey.java

@@ -115,6 +115,7 @@ public boolean start(Session session) throws Exception {
     }
   }
 
+  @SuppressWarnings("fallthrough")
   private boolean _start(Session session, List<Identity> identities, List<String> pkmethods,
       List<String> not_available_pks) throws Exception {
     if (session.auth_failures >= session.max_auth_tries) {
@@ -124,17 +125,40 @@ private boolean _start(Session session, List<Identity> identities, List<String>
     boolean use_pk_auth_query = session.getConfig("enable_pubkey_auth_query").equals("yes");
     boolean try_other_pkmethods =
         session.getConfig("try_additional_pubkey_algorithms").equals("yes");
+    boolean use_openssh_rsa_pubkey_order =
+        session.getConfig("use_openssh_rsa_pubkey_order").equals("yes");
+
+    String[] server_sig_algs = session.getServerSigAlgs();
+    boolean has_server_sig_algs = server_sig_algs != null && server_sig_algs.length > 0;
 
     List<String> rsamethods = new ArrayList<>();
     List<String> nonrsamethods = new ArrayList<>();
     for (String pkmethod : pkmethods) {
-      if (pkmethod.equals("ssh-rsa") || pkmethod.equals("rsa-sha2-256")
-          || pkmethod.equals("rsa-sha2-512") || pkmethod.equals("[email protected]")
-          || pkmethod.equals("[email protected]") || pkmethod.equals("[email protected]")
-          || pkmethod.equals("[email protected]")) {
-        rsamethods.add(pkmethod);
-      } else {
-        nonrsamethods.add(pkmethod);
+      switch (pkmethod) {
+        case "ssh-rsa":
+          if (!has_server_sig_algs && use_openssh_rsa_pubkey_order) {
+            // "Servers that accept rsa-sha2-* signatures for client authentication
+            // SHOULD implement the extension negotiation mechanism defined in
+            // [RFC8308], including especially the "server-sig-algs" extension."
+            //
+            // OpenSSH 8.0 and newer implementations will only attempt the "ssh-rsa" signature
+            // algorithm. To match this behaviour Jsch will force the first attempt, irrespective of
+            // the clients PubkeyAcceptedKeyTypes order, to the "ssh-rsa" signature algorithm. Any
+            // subsequent retries will respect the order as defined by the client.
+            rsamethods.add(0, pkmethod);
+            break;
+          }
+        case "rsa-sha2-256":
+        case "rsa-sha2-512":
+        case "[email protected]":
+        case "[email protected]":
+        case "[email protected]":
+        case "[email protected]":
+          rsamethods.add(pkmethod);
+          break;
+        default:
+          nonrsamethods.add(pkmethod);
+          break;
       }
     }