Scalafix command for scala-cli with basic options and tests (#2968)

* Scalafix command from scala-cli with basic optionns and tests

* Refactor documentation and minor inconveniences

* Make scalac options and scala version configurable

* Remove fixed scalac options

* Correct ScalafixOptions tags to experimental

* Add more tests and fix external rule dependencies forwarding to scalafix

* Differentiate tests for 2.12, 2.13 and 3+ scala versions

* Fix undeleted printlns

* Fmt and fix

* Native image support

* scalafix: fix native image support

* scalafix: run fix + fix tests

* scalafix: add scalafix.dep, more tests

* scalafix: gen doc

* scalafix: support ExplicitResultTypes for scala3 + minor fixes

* scalafix: fix docs tests

---------

Co-authored-by: Vadim Chelyshov <[email protected]>

build.sc

@@ -274,6 +274,13 @@ object dummy extends Module {
       Deps.scalaPy
     )
   }
+  object scalafix extends ScalaModule with Bloop.Module {
+    def skipBloop    = true
+    def scalaVersion = Scala.defaultInternal
+    def ivyDeps = Agg(
+      Deps.scalafixInterfaces
+    )
+  }
 }
 
 trait BuildMacros extends ScalaCliCrossSbtModule
@@ -528,6 +535,8 @@ trait Core extends ScalaCliCrossSbtModule
          |  def mavenAppArtifactId = "${Deps.Versions.mavenAppArtifactId}"
          |  def mavenAppGroupId = "${Deps.Versions.mavenAppGroupId}"
          |  def mavenAppVersion = "${Deps.Versions.mavenAppVersion}"
+         |
+         |  def scalafixVersion = "${Deps.Versions.scalafix}"
          |}
          |""".stripMargin
     if (!os.isFile(dest) || os.read(dest) != code)
@@ -919,7 +928,8 @@ trait Cli extends CrossSbtModule with ProtoBuildModule with CliLaunchers
     Deps.scalaPackager.exclude("com.lihaoyi" -> "os-lib_2.13"),
     Deps.signingCli.exclude((organization, "config_2.13")),
     Deps.slf4jNop, // to silence jgit
-    Deps.sttp
+    Deps.sttp,
+    Deps.scalafixInterfaces
   )
   def compileIvyDeps = super.compileIvyDeps() ++ Agg(
     Deps.jsoniterMacros,

modules/build/src/main/scala/scala/build/ScalafixArtifacts.scala

@@ -0,0 +1,157 @@
+package scala.build
+
+import coursier.cache.FileCache
+import coursier.core.{Repository, Version}
+import coursier.error.{CoursierError, ResolutionError}
+import coursier.util.Task
+import dependency.*
+import org.apache.commons.compress.archivers.zip.ZipFile
+import os.Path
+
+import java.io.ByteArrayInputStream
+import java.util.Properties
+
+import scala.build.EitherCps.{either, value}
+import scala.build.errors.{BuildException, FetchingDependenciesError}
+import scala.build.internal.Constants
+import scala.build.internal.CsLoggerUtil.*
+
+final case class ScalafixArtifacts(
+  scalafixJars: Seq[os.Path],
+  toolsJars: Seq[os.Path]
+)
+
+object ScalafixArtifacts {
+
+  def artifacts(
+    scalaVersion: String,
+    externalRulesDeps: Seq[Positioned[AnyDependency]],
+    extraRepositories: Seq[Repository],
+    logger: Logger,
+    cache: FileCache[Task]
+  ): Either[BuildException, ScalafixArtifacts] =
+    either {
+      val scalafixProperties =
+        value(fetchOrLoadScalafixProperties(extraRepositories, logger, cache))
+      val key =
+        value(scalafixPropsKey(scalaVersion))
+      val fetchScalaVersion = scalafixProperties.getProperty(key)
+
+      val scalafixDeps =
+        Seq(dep"ch.epfl.scala:scalafix-cli_$fetchScalaVersion:${Constants.scalafixVersion}")
+
+      val scalafix =
+        value(
+          Artifacts.artifacts(
+            scalafixDeps.map(Positioned.none),
+            extraRepositories,
+            None,
+            logger,
+            cache.withMessage(s"Downloading scalafix-cli ${Constants.scalafixVersion}")
+          )
+        )
+
+      val scalaParameters =
+        // Scalafix for scala 3 uses 2.13-published community rules
+        // https://github.com/scalacenter/scalafix/issues/2041
+        if (scalaVersion.startsWith("3")) ScalaParameters(Constants.defaultScala213Version)
+        else ScalaParameters(scalaVersion)
+
+      val tools =
+        value(
+          Artifacts.artifacts(
+            externalRulesDeps,
+            extraRepositories,
+            Some(scalaParameters),
+            logger,
+            cache.withMessage(s"Downloading scalafix.deps")
+          )
+        )
+
+      ScalafixArtifacts(scalafix.map(_._2), tools.map(_._2))
+    }
+
+  private def fetchOrLoadScalafixProperties(
+    extraRepositories: Seq[Repository],
+    logger: Logger,
+    cache: FileCache[Task]
+  ): Either[BuildException, Properties] =
+    either {
+      val cacheDir  = Directories.directories.cacheDir / "scalafix-props-cache"
+      val cachePath = cacheDir / s"scalafix-interfaces-${Constants.scalafixVersion}.properties"
+
+      val content =
+        if (!os.exists(cachePath)) {
+          val interfacesJar = value(fetchScalafixInterfaces(extraRepositories, logger, cache))
+          val propsData     = value(readScalafixProperties(interfacesJar))
+          if (!os.exists(cacheDir)) os.makeDir(cacheDir)
+          os.write(cachePath, propsData)
+          propsData
+        }
+        else os.read(cachePath)
+      val props  = new Properties()
+      val stream = new ByteArrayInputStream(content.getBytes())
+      props.load(stream)
+      props
+    }
+
+  private def fetchScalafixInterfaces(
+    extraRepositories: Seq[Repository],
+    logger: Logger,
+    cache: FileCache[Task]
+  ): Either[BuildException, Path] =
+    either {
+      val scalafixInterfaces = dep"ch.epfl.scala:scalafix-interfaces:${Constants.scalafixVersion}"
+
+      val fetchResult =
+        value(
+          Artifacts.artifacts(
+            List(scalafixInterfaces).map(Positioned.none),
+            extraRepositories,
+            None,
+            logger,
+            cache.withMessage(s"Downloading scalafix-interfaces ${scalafixInterfaces.version}")
+          )
+        )
+
+      val expectedJarName = s"scalafix-interfaces-${Constants.scalafixVersion}.jar"
+      val interfacesJar = fetchResult.collectFirst {
+        case (_, path) if path.last == expectedJarName => path
+      }
+
+      value(
+        interfacesJar.toRight(new BuildException("Failed to found scalafix-interfaces jar") {})
+      )
+    }
+
+  private def readScalafixProperties(jar: Path): Either[BuildException, String] = {
+    import scala.jdk.CollectionConverters.*
+    val zipFile = new ZipFile(jar.toNIO)
+    val entry = zipFile.getEntries().asScala.find(entry =>
+      entry.getName() == "scalafix-interfaces.properties"
+    )
+    val out =
+      entry.toRight(new BuildException("Failed to found scalafix properties") {})
+        .map { entry =>
+          val stream = zipFile.getInputStream(entry)
+          val bytes  = stream.readAllBytes()
+          new String(bytes)
+        }
+    zipFile.close()
+    out
+  }
+
+  private def scalafixPropsKey(scalaVersion: String): Either[BuildException, String] = {
+    val regex = "(\\d)\\.(\\d+).+".r
+    scalaVersion match {
+      case regex("2", "12")              => Right("scala212")
+      case regex("2", "13")              => Right("scala213")
+      case regex("3", x) if x.toInt <= 3 => Right("scala3LTS")
+      case regex("3", _)                 => Right("scala3Next")
+      case _ =>
+        Left(new BuildException(s"Scalafix is not supported for Scala version: $scalaVersion") {})
+    }
+
+  }
+
+}

modules/cli/src/main/scala/scala/cli/ScalaCliCommands.scala

@@ -39,6 +39,7 @@ class ScalaCliCommands(
     export0.Export,
     fix.Fix,
     fmt.Fmt,
+    scalafix.Scalafix,
     new HelpCmd(help),
     installcompletions.InstallCompletions,
     installhome.InstallHome,

modules/cli/src/main/scala/scala/cli/commands/scalafix/Scalafix.scala

@@ -0,0 +1,150 @@
+package scala.cli.commands.scalafix
+
+import caseapp.*
+import caseapp.core.help.HelpFormat
+import coursier.cache.FileCache
+import dependency.*
+import scalafix.interfaces.ScalafixError.*
+import scalafix.interfaces.{
+  Scalafix => ScalafixInterface,
+  ScalafixError,
+  ScalafixException,
+  ScalafixRule
+}
+
+import java.io.File
+import java.util.Optional
+
+import scala.build.EitherCps.{either, value}
+import scala.build.input.{Inputs, Script, SourceScalaFile}
+import scala.build.internal.{Constants, ExternalBinaryParams, FetchExternalBinary, Runner}
+import scala.build.options.{BuildOptions, Scope}
+import scala.build.{Artifacts, Build, BuildThreads, Logger, ScalafixArtifacts, Sources}
+import scala.cli.CurrentParams
+import scala.cli.commands.compile.Compile.buildOptionsOrExit
+import scala.cli.commands.fmt.FmtUtil.*
+import scala.cli.commands.shared.{HelpCommandGroup, HelpGroup, SharedOptions}
+import scala.cli.commands.{ScalaCommand, SpecificationLevel, compile}
+import scala.cli.config.Keys
+import scala.cli.util.ArgHelpers.*
+import scala.cli.util.ConfigDbUtils
+import scala.collection.mutable
+import scala.collection.mutable.Buffer
+import scala.jdk.CollectionConverters.*
+import scala.jdk.OptionConverters.*
+
+object Scalafix extends ScalaCommand[ScalafixOptions] {
+  override def group: String = HelpCommandGroup.Main.toString
+  override def sharedOptions(options: ScalafixOptions): Option[SharedOptions] = Some(options.shared)
+  override def scalaSpecificationLevel: SpecificationLevel = SpecificationLevel.EXPERIMENTAL
+
+  val hiddenHelpGroups: Seq[HelpGroup] =
+    Seq(
+      HelpGroup.Scala,
+      HelpGroup.Java,
+      HelpGroup.Dependency,
+      HelpGroup.ScalaJs,
+      HelpGroup.ScalaNative,
+      HelpGroup.CompilationServer,
+      HelpGroup.Debug
+    )
+  override def helpFormat: HelpFormat = super.helpFormat
+    .withHiddenGroups(hiddenHelpGroups)
+    .withHiddenGroupsWhenShowHidden(hiddenHelpGroups)
+    .withPrimaryGroup(HelpGroup.Format)
+  override def names: List[List[String]] = List(
+    List("scalafix")
+  )
+
+  override def runCommand(options: ScalafixOptions, args: RemainingArgs, logger: Logger): Unit = {
+    val buildOptions = buildOptionsOrExit(options)
+    val buildOptionsWithSemanticDb = buildOptions.copy(scalaOptions =
+      buildOptions.scalaOptions.copy(semanticDbOptions =
+        buildOptions.scalaOptions.semanticDbOptions.copy(generateSemanticDbs = Some(true))
+      )
+    )
+    val inputs        = options.shared.inputs(args.all).orExit(logger)
+    val threads       = BuildThreads.create()
+    val compilerMaker = options.shared.compilerMaker(threads)
+    val configDb      = ConfigDbUtils.configDb.orExit(logger)
+    val actionableDiagnostics =
+      options.shared.logging.verbosityOptions.actions.orElse(
+        configDb.get(Keys.actions).getOrElse(None)
+      )
+
+    val workspace =
+      if (args.all.isEmpty) os.pwd
+      else inputs.workspace
+
+    val scalaVersion =
+      options.buildOptions.orExit(logger).scalaParams.orExit(logger).map(_.scalaVersion)
+        .getOrElse(Constants.defaultScalaVersion)
+    val scalaBinVersion =
+      options.buildOptions.orExit(logger).scalaParams.orExit(logger).map(_.scalaBinaryVersion)
+
+    val configFilePathOpt = options.scalafixConf.map(os.Path(_, os.pwd))
+
+    val res = Build.build(
+      inputs,
+      buildOptionsWithSemanticDb,
+      compilerMaker,
+      None,
+      logger,
+      crossBuilds = false,
+      buildTests = false,
+      partial = None,
+      actionableDiagnostics = actionableDiagnostics
+    )
+    val builds = res.orExit(logger)
+
+    builds.get(Scope.Main).flatMap(_.successfulOpt) match
+      case None => sys.exit(1)
+      case Some(build) =>
+        val classPaths = build.fullClassPath
+
+        val scalacOptions = options.shared.scalac.scalacOption ++
+          build.options.scalaOptions.scalacOptions.toSeq.map(_.value.value)
+
+        either {
+          val artifacts =
+            value(
+              ScalafixArtifacts.artifacts(
+                scalaVersion,
+                build.options.classPathOptions.scalafixDependencies.values.flatten,
+                value(buildOptions.finalRepositories),
+                logger,
+                buildOptions.internal.cache.getOrElse(FileCache())
+              )
+            )
+
+          val scalafixOptions =
+            options.scalafixConf.toList.flatMap(scalafixConf => List("--config", scalafixConf)) ++
+              Seq("--sourceroot", workspace.toString) ++
+              Seq("--classpath", classPaths.mkString(java.io.File.pathSeparator)) ++
+              Seq("--scala-version", scalaVersion) ++
+              (if (options.check) Seq("--test") else Nil) ++
+              (if (scalacOptions.nonEmpty) scalacOptions.flatMap(Seq("--scalac-options", _))
+               else Nil) ++
+              (if (artifacts.toolsJars.nonEmpty)
+                 Seq("--tool-classpath", artifacts.toolsJars.mkString(java.io.File.pathSeparator))
+               else Nil) ++
+              options.rules.flatMap(Seq("-r", _))
+              ++ options.scalafixArg
+
+          val proc = Runner.runJvm(
+            buildOptions.javaHome().value.javaCommand,
+            buildOptions.javaOptions.javaOpts.toSeq.map(_.value.value),
+            artifacts.scalafixJars,
+            "scalafix.cli.Cli",
+            scalafixOptions,
+            logger,
+            cwd = Some(workspace),
+            allowExecve = true
+          )
+
+          sys.exit(proc.waitFor())
+        }
+
+  }
+
+}