From cc73322fe3c8bf93ce10ad67b7388d6f301e8447 Mon Sep 17 00:00:00 2001 From: Maciej Gajek Date: Mon, 2 Oct 2023 15:05:32 +0200 Subject: [PATCH] WIP --- .../scala/cli/commands/publish/Publish.scala | 23 ++++++-- .../publish/PublishConnectionOptions.scala | 46 ++++++++++++++++ .../cli/commands/publish/PublishLocal.scala | 1 + .../cli/commands/publish/PublishOptions.scala | 2 + .../cli/commands/publish/RepoParams.scala | 53 ++++++++++++++++--- .../commands/util/ScalaCliSttpBackend.scala | 13 +++-- .../options/PublishContextualOptions.scala | 7 ++- website/docs/reference/cli-options.md | 33 ++++++++++++ website/docs/reference/commands.md | 2 +- 9 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 modules/cli/src/main/scala/scala/cli/commands/publish/PublishConnectionOptions.scala diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/Publish.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/Publish.scala index d0811f1a65..a380ce22db 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/publish/Publish.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/publish/Publish.scala @@ -55,6 +55,7 @@ import scala.cli.commands.shared.{ } import scala.cli.commands.util.{BuildCommandHelpers, ScalaCliSttpBackend} import scala.cli.commands.{ScalaCommand, SpecificationLevel, WatchUtil} +import scala.concurrent.duration.DurationInt import scala.cli.config.{ConfigDb, Keys, PasswordOption, PublishCredentials} import scala.cli.errors.{ FailedToSignFileError, @@ -91,6 +92,7 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers { sharedPublish: SharedPublishOptions, publishRepo: PublishRepositoryOptions, scalaSigning: PgpScalaSigningOptions, + publishConnection: PublishConnectionOptions, mainClass: MainClassOptions, ivy2LocalLike: Option[Boolean] ): Either[BuildException, BuildOptions] = either { @@ -122,7 +124,12 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers { val input = sharedPublish.checksum.flatMap(_.split(",")).map(_.trim).filter(_.nonEmpty) if (input.isEmpty) None else Some(input) - } + }, + connectionTimeoutRetries = publishConnection.connectionTimeoutRetries, + connectionTimeoutSeconds = publishConnection.connectionTimeoutSeconds, + responseTimeoutSeconds = publishConnection.responseTimeoutSeconds, + stagingRepoRetries = publishConnection.stagingRepoRetries, + stagingRepoWaitTimeMilis = publishConnection.stagingRepoWaitTimeMilis ) baseOptions.copy( mainClass = mainClass.mainClass.filter(_.nonEmpty), @@ -212,6 +219,7 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers { options.sharedPublish, options.publishRepo, options.signingCli, + options.connectionOptions, options.mainClass, options.ivy2LocalLike ).orExit(logger) @@ -819,7 +827,11 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers { ivy2HomeOpt, publishOptions.contextual(isCi).repositoryIsIvy2LocalLike.getOrElse(false), es, - logger + logger, + publishOptions.contextual(isCi).connectionTimeoutRetries, + publishOptions.contextual(isCi).connectionTimeoutSeconds, + publishOptions.contextual(isCi).stagingRepoRetries, + publishOptions.contextual(isCi).stagingRepoWaitTimeMilis ) } } @@ -1043,7 +1055,12 @@ object Publish extends ScalaCommand[PublishOptions] with BuildCommandHelpers { val baseUpload = if (retainedRepo.root.startsWith("http://") || retainedRepo.root.startsWith("https://")) - HttpURLConnectionUpload.create() + HttpURLConnectionUpload.create( + publishOptions.contextual(isCi) + .connectionTimeoutSeconds.map(_.seconds.toMillis.toInt), + publishOptions.contextual(isCi) + .responseTimeoutSeconds.map(_.seconds.toMillis.toInt) + ) else FileUpload(Paths.get(new URI(retainedRepo.root))) diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishConnectionOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishConnectionOptions.scala new file mode 100644 index 0000000000..2c2b92a978 --- /dev/null +++ b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishConnectionOptions.scala @@ -0,0 +1,46 @@ +package scala.cli.commands.publish + +import caseapp.* + +import scala.cli.commands.shared.{HelpGroup, SharedVersionOptions} +import scala.cli.commands.tags + +// format: off +final case class PublishConnectionOptions( + @Group(HelpGroup.Publishing.toString) + @HelpMessage("Connection timeout, in seconds.") + @Tag(tags.restricted) + @Hidden + connectionTimeoutSeconds: Option[Int] = None, + + @Group(HelpGroup.Publishing.toString) + @HelpMessage("How many times to retry establishing the connection on timeout.") + @Tag(tags.restricted) + @Hidden + connectionTimeoutRetries: Option[Int] = None, + + @Group(HelpGroup.Publishing.toString) + @HelpMessage("Waiting for response timeout, in seconds.") + @Tag(tags.restricted) + @Hidden + responseTimeoutSeconds: Option[Int] = None, + + @Group(HelpGroup.Publishing.toString) + @HelpMessage("How many times to retry the staging repository operations on failure.") + @Tag(tags.restricted) + @Hidden + stagingRepoRetries: Option[Int] = None, + + @Group(HelpGroup.Publishing.toString) + @HelpMessage("Time to wait between staging repository operation retries, in milliseconds.") + @Tag(tags.restricted) + @Hidden + stagingRepoWaitTimeMilis: Option[Int] = None + ) { + // format: on +} + +object PublishConnectionOptions { + implicit lazy val parser: Parser[PublishConnectionOptions] = Parser.derive + implicit lazy val help: Help[PublishConnectionOptions] = Help.derive +} diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocal.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocal.scala index 0b9365310c..2c8812583e 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocal.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishLocal.scala @@ -45,6 +45,7 @@ object PublishLocal extends ScalaCommand[PublishLocalOptions] { options.sharedPublish, PublishRepositoryOptions(), options.scalaSigning, + PublishConnectionOptions(), options.mainClass, None ).orExit(logger) diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishOptions.scala index 6095ddb803..f5eaea78be 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishOptions.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishOptions.scala @@ -26,6 +26,8 @@ final case class PublishOptions( sharedPublish: SharedPublishOptions = SharedPublishOptions(), @Recurse signingCli: PgpScalaSigningOptions = PgpScalaSigningOptions(), + @Recurse + connectionOptions: PublishConnectionOptions = PublishConnectionOptions(), @Group(HelpGroup.Publishing.toString) @Tag(tags.restricted) diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/RepoParams.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/RepoParams.scala index 1b6c224235..9f8078c986 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/publish/RepoParams.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/publish/RepoParams.scala @@ -3,10 +3,10 @@ package scala.cli.commands.publish import coursier.core.Authentication import coursier.maven.MavenRepository import coursier.publish.sonatype.SonatypeApi +import coursier.publish.util.EmaRetryParams import coursier.publish.{Hooks, PublishRepository} import java.util.concurrent.ScheduledExecutorService - import scala.build.EitherCps.{either, value} import scala.build.Logger import scala.build.errors.BuildException @@ -50,15 +50,35 @@ object RepoParams { ivy2HomeOpt: Option[os.Path], isIvy2LocalLike: Boolean, es: ScheduledExecutorService, - logger: Logger + logger: Logger, + connectionTimeoutRetries: Option[Int] = None, + connectionTimeoutSeconds: Option[Int] = None, + stagingRepoRetries: Option[Int] = None, + stagingRepoWaitTimeMilis: Option[Int] = None ): Either[BuildException, RepoParams] = either { repo match { case "ivy2-local" => RepoParams.ivy2Local(ivy2HomeOpt) case "sonatype" | "central" | "maven-central" | "mvn-central" => - RepoParams.centralRepo("https://oss.sonatype.org", es, logger) + RepoParams.centralRepo( + "https://oss.sonatype.org", + connectionTimeoutRetries, + connectionTimeoutSeconds, + stagingRepoRetries, + stagingRepoWaitTimeMilis, + es, + logger + ) case "sonatype-s01" | "central-s01" | "maven-central-s01" | "mvn-central-s01" => - RepoParams.centralRepo("https://s01.oss.sonatype.org", es, logger) + RepoParams.centralRepo( + "https://s01.oss.sonatype.org", + connectionTimeoutRetries, + connectionTimeoutSeconds, + stagingRepoRetries, + stagingRepoWaitTimeMilis, + es, + logger + ) case "github" => value(RepoParams.gitHubRepo(vcsUrlOpt, workspace, logger)) case repoStr if repoStr.startsWith("github:") && repoStr.count(_ == '/') == 1 => @@ -89,10 +109,29 @@ object RepoParams { } } - def centralRepo(base: String, es: ScheduledExecutorService, logger: Logger) = { + def centralRepo( + base: String, + connectionTimeoutRetries: Option[Int], + connectionTimeoutSeconds: Option[Int], + stagingRepoRetries: Option[Int], + stagingRepoWaitTimeMilis: Option[Int], + es: ScheduledExecutorService, + logger: Logger + ) = { val repo0 = PublishRepository.Sonatype(MavenRepository(base)) - val backend = ScalaCliSttpBackend.httpURLConnection(logger) - val api = SonatypeApi(backend, base + "/service/local", None, logger.verbosity) + val backend = ScalaCliSttpBackend.httpURLConnection(logger, connectionTimeoutSeconds) + val api = SonatypeApi( + backend, + base + "/service/local", + None, + logger.verbosity, + retryOnTimeout = connectionTimeoutRetries.getOrElse(3), + stagingRepoRetryParams = EmaRetryParams( + stagingRepoRetries.getOrElse(3), + stagingRepoWaitTimeMilis.getOrElse(10 * 1000), + 2.0f + ) + ) val hooks0 = Hooks.sonatype( repo0, api, diff --git a/modules/cli/src/main/scala/scala/cli/commands/util/ScalaCliSttpBackend.scala b/modules/cli/src/main/scala/scala/cli/commands/util/ScalaCliSttpBackend.scala index d5966f3f39..8a6e2abd28 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/util/ScalaCliSttpBackend.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/util/ScalaCliSttpBackend.scala @@ -1,10 +1,12 @@ package scala.cli.commands.util import sttp.capabilities.Effect -import sttp.client3._ +import sttp.client3.* import sttp.monad.MonadError import scala.build.Logger +import scala.concurrent.duration +import scala.concurrent.duration.FiniteDuration import scala.util.Try class ScalaCliSttpBackend( @@ -38,9 +40,14 @@ class ScalaCliSttpBackend( } object ScalaCliSttpBackend { - def httpURLConnection(logger: Logger): ScalaCliSttpBackend = + def httpURLConnection(logger: Logger, timeoutSeconds: Option[Int] = None): ScalaCliSttpBackend = new ScalaCliSttpBackend( - HttpURLConnectionBackend(), + HttpURLConnectionBackend( + options = timeoutSeconds + .fold(SttpBackendOptions.Default)(t => + SttpBackendOptions.connectionTimeout(FiniteDuration(t, duration.SECONDS)) + ) + ), logger ) } diff --git a/modules/options/src/main/scala/scala/build/options/PublishContextualOptions.scala b/modules/options/src/main/scala/scala/build/options/PublishContextualOptions.scala index cb862aba1a..5a8118e6bc 100644 --- a/modules/options/src/main/scala/scala/build/options/PublishContextualOptions.scala +++ b/modules/options/src/main/scala/scala/build/options/PublishContextualOptions.scala @@ -19,7 +19,12 @@ final case class PublishContextualOptions( repoPassword: Option[PasswordOption] = None, repoRealm: Option[String] = None, computeVersion: Option[ComputeVersion] = None, - checksums: Option[Seq[String]] = None + checksums: Option[Seq[String]] = None, + connectionTimeoutRetries: Option[Int] = None, + connectionTimeoutSeconds: Option[Int] = None, + responseTimeoutSeconds: Option[Int] = None, + stagingRepoRetries: Option[Int] = None, + stagingRepoWaitTimeMilis: Option[Int] = None ) object PublishContextualOptions { diff --git a/website/docs/reference/cli-options.md b/website/docs/reference/cli-options.md index 196819983e..415f0f0611 100644 --- a/website/docs/reference/cli-options.md +++ b/website/docs/reference/cli-options.md @@ -2092,6 +2092,39 @@ Available in commands: ### `--key` [Internal] +### Publish connection options + +Available in commands: + +[`publish`](./commands.md#publish) + + + +### `--connection-timeout-seconds` + +[Internal] +Connection timeout, in seconds. + +### `--connection-timeout-retries` + +[Internal] +How many times to retry establishing the connection on timeout. + +### `--response-timeout-seconds` + +[Internal] +Waiting for response timeout, in seconds. + +### `--staging-repo-retries` + +[Internal] +How many times to retry the staging repository operations on failure. + +### `--staging-repo-wait-time-milis` + +[Internal] +Time to wait between staging repository operation retries, in milliseconds. + ### Setup IDE options Available in commands: diff --git a/website/docs/reference/commands.md b/website/docs/reference/commands.md index 4ea9c14c83..313b89cad9 100644 --- a/website/docs/reference/commands.md +++ b/website/docs/reference/commands.md @@ -251,7 +251,7 @@ The `publish` sub-command is experimental. Please bear in mind that non-ideal user experience should be expected. If you encounter any bugs or have feedback to share, make sure to reach out to the maintenance team at https://github.com/VirtusLab/scala-cli -Accepts option groups: [compilation server](./cli-options.md#compilation-server-options), [coursier](./cli-options.md#coursier-options), [cross](./cli-options.md#cross-options), [debug](./cli-options.md#debug-options), [dependency](./cli-options.md#dependency-options), [global suppress warning](./cli-options.md#global-suppress-warning-options), [help group](./cli-options.md#help-group-options), [input](./cli-options.md#input-options), [jvm](./cli-options.md#jvm-options), [logging](./cli-options.md#logging-options), [main class](./cli-options.md#main-class-options), [markdown](./cli-options.md#markdown-options), [pgp scala signing](./cli-options.md#pgp-scala-signing-options), [publish](./cli-options.md#publish-options), [publish params](./cli-options.md#publish-params-options), [publish repository](./cli-options.md#publish-repository-options), [python](./cli-options.md#python-options), [Scala.js](./cli-options.md#scalajs-options), [Scala Native](./cli-options.md#scala-native-options), [scalac](./cli-options.md#scalac-options), [scalac extra](./cli-options.md#scalac-extra-options), [shared](./cli-options.md#shared-options), [snippet](./cli-options.md#snippet-options), [source generator](./cli-options.md#source-generator-options), [suppress warning](./cli-options.md#suppress-warning-options), [verbosity](./cli-options.md#verbosity-options), [version](./cli-options.md#version-options), [watch](./cli-options.md#watch-options), [workspace](./cli-options.md#workspace-options) +Accepts option groups: [compilation server](./cli-options.md#compilation-server-options), [coursier](./cli-options.md#coursier-options), [cross](./cli-options.md#cross-options), [debug](./cli-options.md#debug-options), [dependency](./cli-options.md#dependency-options), [global suppress warning](./cli-options.md#global-suppress-warning-options), [help group](./cli-options.md#help-group-options), [input](./cli-options.md#input-options), [jvm](./cli-options.md#jvm-options), [logging](./cli-options.md#logging-options), [main class](./cli-options.md#main-class-options), [markdown](./cli-options.md#markdown-options), [pgp scala signing](./cli-options.md#pgp-scala-signing-options), [publish](./cli-options.md#publish-options), [publish connection](./cli-options.md#publish-connection-options), [publish params](./cli-options.md#publish-params-options), [publish repository](./cli-options.md#publish-repository-options), [python](./cli-options.md#python-options), [Scala.js](./cli-options.md#scalajs-options), [Scala Native](./cli-options.md#scala-native-options), [scalac](./cli-options.md#scalac-options), [scalac extra](./cli-options.md#scalac-extra-options), [shared](./cli-options.md#shared-options), [snippet](./cli-options.md#snippet-options), [source generator](./cli-options.md#source-generator-options), [suppress warning](./cli-options.md#suppress-warning-options), [verbosity](./cli-options.md#verbosity-options), [version](./cli-options.md#version-options), [watch](./cli-options.md#watch-options), [workspace](./cli-options.md#workspace-options) ## publish local