Require all packages to solve / compile and include all valid compilers in their metadata

SPEC.md

@@ -207,7 +207,7 @@ Note:
 
 - Globs you provide at the `includeFiles` and `excludeFiles` keys must contain only `*`, `**`, `/`, `.`, `..`, and characters for Linux file paths. It is not possible to negate a glob (ie. the `!` character), and globs cannot represent a path out of the package source directory.
 - When packaging your project source, the registry will first "include" your `src` directory and always-included files such as your `purs.json` file. Then it will include files which match globs indicated by the `includeFiles` key ([always-ignored files](#always-ignored-files) cannot be included). Finally, it will apply the excluding globs indicated by the `excludeFiles` key to the included files ([always-included files](#always-included-files) cannot be excluded).
-- Dependencies you provide at the `dependencies` key must exist in the registry, and the dependency ranges must be solvable (ie. it must be possible to produce a single version of each dependency that satisfies the provided version bounds, including any transitive dependencies).
+- Dependencies you provide at the `dependencies` key must exist in the registry, the dependency ranges must be solvable (ie. it must be possible to produce a single version of each dependency that satisfies the provided version bounds, including any transitive dependencies), and transitive dependencies are not allowed (ie. any modules you import in your code must come from packages listed in your dependencies).
 
 For example:
 
@@ -234,11 +234,12 @@ For example:
 
 All packages in the registry have an associated metadata file, which is located in the `metadata` directory of the `registry` repository under the package name. For example, the metadata for the `aff` package is located at: https://github.com/purescript/registry/blob/main/metadata/aff.json. Metadata files are the source of truth on all published and unpublished versions for a particular package for what there content is and where the package is located. Metadata files are produced by the registry, not by package authors, though they take some information from package manifests.
 
-Each published version of a package records three fields:
+Each published version of a package records four fields:
 
 - `hash`: a [`Sha256`](#Sha256) of the compressed archive fetched by the registry for the given version
 - `bytes`: the size of the tarball in bytes
 - `publishedTime`: the time the package was published as an `ISO8601` string
+- `compilers`: compiler versions this package is known to work with. This field can be in one of two states: a single version indicates that the package worked with a specific compiler on upload but has not yet been tested with all compilers, whereas a non-empty array of versions indicates the package has been tested with all compilers the registry supports.
 
 Each unpublished version of a package records three fields:
 

app/fixtures/github-packages/transitive-1.0.0/bower.json

@@ -0,0 +1,12 @@
+{
+  "name": "purescript-transitive",
+  "homepage": "https://github.com/purescript/purescript-transitive",
+  "license": "BSD-3-Clause",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/purescript/purescript-transitive.git"
+  },
+  "dependencies": {
+    "purescript-effect": "^4.0.0"
+  }
+}

app/fixtures/github-packages/transitive-1.0.0/src/Transitive.purs

@@ -0,0 +1,6 @@
+module Transitive where
+
+import Prelude
+
+uno :: Int
+uno = one

app/fixtures/registry/metadata/prelude.json

@@ -6,6 +6,7 @@
   "published": {
     "6.0.1": {
       "bytes": 31142,
+      "compilers": ["0.15.3", "0.15.4", "0.15.5"],
       "hash": "sha256-o8p6SLYmVPqzXZhQFd2hGAWEwBoXl1swxLG/scpJ0V0=",
       "publishedTime": "2022-08-18T20:04:00.000Z",
       "ref": "v6.0.1"

app/fixtures/registry/metadata/type-equality.json

@@ -6,6 +6,7 @@
   "published": {
     "4.0.1": {
       "bytes": 2184,
+      "compilers": ["0.15.2", "0.15.3", "0.15.4"],
       "hash": "sha256-Hs9D6Y71zFi/b+qu5NSbuadUQXe5iv5iWx0226vOHUw=",
       "publishedTime": "2022-04-27T18:00:18.000Z",
       "ref": "v4.0.1"

app/src/App/CLI/Git.purs

@@ -111,11 +111,12 @@ gitPull { address: { owner, repo }, pullMode } cwd = Except.runExcept do
         , " has no untracked or dirty files, it is safe to pull the latest."
         ]
       pure true
-    Just files -> do
-      Log.debug $ Array.fold
-        [ "Some files are untracked or dirty in local checkout of " <> cwd <> ": "
-        , NonEmptyArray.foldMap1 (append "\n  - ") files
-        ]
+    Just _files -> do
+      -- This is a bit noisy, so commenting it out for now.
+      -- Log.debug $ Array.fold
+      --   [ "Some files are untracked or dirty in local checkout of " <> cwd <> ": "
+      --   , NonEmptyArray.foldMap1 (append "\n  - ") files
+      --   ]
       Log.warn $ Array.fold
         [ "Local checkout of " <> formatted
         , " has untracked or dirty files, it may not be safe to pull the latest."

app/src/App/CLI/Purs.purs

@@ -4,6 +4,7 @@ import Registry.App.Prelude
 
 import Codec.JSON.DecodeError as CJ.DecodeError
 import Data.Array as Array
+import Data.Codec as Codec
 import Data.Codec.JSON as CJ
 import Data.Codec.JSON.Common as CJ.Common
 import Data.Codec.JSON.Record as CJ.Record
@@ -13,6 +14,16 @@ import Node.ChildProcess.Types (Exit(..))
 import Node.Library.Execa as Execa
 import Registry.Version as Version
 
+-- | The minimum compiler version that supports 'purs graph'
+minPursGraph :: Version
+minPursGraph = unsafeFromRight (Version.parse "0.14.0")
+
+minPursuitPublish :: Version
+minPursuitPublish = unsafeFromRight (Version.parse "0.14.7")
+
+minLanguageCSTParser :: Version
+minLanguageCSTParser = unsafeFromRight (Version.parse "0.15.0")
+
 -- | Call a specific version of the PureScript compiler
 callCompiler_ :: { version :: Maybe Version, command :: PursCommand, cwd :: Maybe FilePath } -> Aff Unit
 callCompiler_ = void <<< callCompiler
@@ -23,6 +34,22 @@ data CompilerFailure
   | MissingCompiler
 
 derive instance Eq CompilerFailure
+derive instance Ord CompilerFailure
+
+compilerFailureCodec :: CJ.Codec CompilerFailure
+compilerFailureCodec = Codec.codec' decode encode
+  where
+  decode :: JSON -> Except CJ.DecodeError CompilerFailure
+  decode json = except do
+    map CompilationError (CJ.decode (CJ.array compilerErrorCodec) json)
+      <|> map UnknownError (CJ.decode CJ.string json)
+      <|> map (const MissingCompiler) (CJ.decode CJ.null json)
+
+  encode :: CompilerFailure -> JSON
+  encode = case _ of
+    CompilationError errors -> CJ.encode (CJ.array compilerErrorCodec) errors
+    UnknownError message -> CJ.encode CJ.string message
+    MissingCompiler -> CJ.encode CJ.null unit
 
 type CompilerError =
   { position :: SourcePosition

app/src/App/Effect/Cache.purs

@@ -169,7 +169,6 @@ handleMemoryFs env = case _ of
           case inFs of
             Nothing -> pure $ reply Nothing
             Just entry -> do
-              Log.debug $ "Fell back to on-disk entry for " <> memory
               putMemoryImpl env.ref unit (Key memory (Const entry))
               pure $ reply $ Just $ unCache entry
         Just cached ->
@@ -227,8 +226,7 @@ getMemoryImpl ref (Key id (Reply reply)) = do
   let (unCache :: CacheValue -> b) = unsafeCoerce
   cache <- Run.liftEffect $ Ref.read ref
   case Map.lookup id cache of
-    Nothing -> do
-      Log.debug $ "No cache entry found for " <> id <> " in memory."
+    Nothing ->
       pure $ reply Nothing
     Just cached -> do
       pure $ reply $ Just $ unCache cached
@@ -237,7 +235,6 @@ putMemoryImpl :: forall x r a. CacheRef -> a -> MemoryEncoding Const a x -> Run
 putMemoryImpl ref next (Key id (Const value)) = do
   let (toCache :: x -> CacheValue) = unsafeCoerce
   Run.liftEffect $ Ref.modify_ (Map.insert id (toCache value)) ref
-  Log.debug $ "Wrote cache entry for " <> id <> " in memory."
   pure next
 
 deleteMemoryImpl :: forall x r a. CacheRef -> MemoryEncoding Ignore a x -> Run (LOG + EFFECT + r) a
@@ -276,7 +273,6 @@ getFsImpl cacheDir = case _ of
     let path = Path.concat [ cacheDir, safePath id ]
     Run.liftAff (Aff.attempt (FS.Aff.readFile path)) >>= case _ of
       Left _ -> do
-        Log.debug $ "No cache found for " <> id <> " at path " <> path
         pure $ reply Nothing
       Right buf -> do
         pure $ reply $ Just buf
@@ -285,7 +281,6 @@ getFsImpl cacheDir = case _ of
     let path = Path.concat [ cacheDir, safePath id ]
     Run.liftAff (Aff.attempt (FS.Aff.readTextFile UTF8 path)) >>= case _ of
       Left _ -> do
-        Log.debug $ "No cache file found for " <> id <> " at path " <> path
         pure $ reply Nothing
       Right content -> case JSON.parse content of
         Left parseError -> do
@@ -308,7 +303,6 @@ putFsImpl cacheDir next = case _ of
         Log.warn $ "Failed to write cache entry for " <> id <> " at path " <> path <> " as a buffer: " <> Aff.message fsError
         pure next
       Right _ -> do
-        Log.debug $ "Wrote cache entry for " <> id <> " as a buffer at path " <> path
         pure next
 
   AsJson id codec (Const value) -> do
@@ -318,7 +312,6 @@ putFsImpl cacheDir next = case _ of
         Log.warn $ "Failed to write cache entry for " <> id <> " at path " <> path <> " as JSON: " <> Aff.message fsError
         pure next
       Right _ -> do
-        Log.debug $ "Wrote cache entry for " <> id <> " at path " <> path <> " as JSON."
         pure next
 
 deleteFsImpl :: forall a b r. FilePath -> FsEncoding Ignore a b -> Run (LOG + AFF + r) a

app/src/App/Effect/GitHub.purs

@@ -242,8 +242,8 @@ request octokit githubRequest@{ route: route@(GitHubRoute method _ _), codec } =
             -- auto-expire cache entries. We will be behind GitHub at most this amount per repo.
             --
             -- TODO: This 'diff' check should be removed once we have conditional requests.
-            Right _ | DateTime.diff now prevResponse.modified >= Duration.Hours 4.0 -> do
-              Log.debug $ "Found cache entry but it was modified more than 4 hours ago, refetching " <> printedRoute
+            Right _ | DateTime.diff now prevResponse.modified >= Duration.Hours 23.0 -> do
+              Log.debug $ "Found cache entry but it was modified more than 23 hours ago, refetching " <> printedRoute
               result <- requestWithBackoff octokit githubRequest
               Cache.put _githubCache (Request route) (result <#> \resp -> { response: CJ.encode codec resp, modified: now, etag: Nothing })
               pure result
@@ -265,10 +265,8 @@ requestWithBackoff octokit githubRequest = do
   Log.debug $ "Making request to " <> route <> " with base URL " <> githubApiUrl
   result <- Run.liftAff do
     let
-      retryOptions =
-        { timeout: defaultRetry.timeout
-        , retryOnCancel: defaultRetry.retryOnCancel
-        , retryOnFailure: \attempt err -> case err of
+      retryOptions = defaultRetry
+        { retryOnFailure = \attempt err -> case err of
             UnexpectedError _ -> false
             DecodeError _ -> false
             -- https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#exceeding-the-rate-limit

app/src/App/Effect/PackageSets.purs

@@ -428,7 +428,7 @@ validatePackageSet (PackageSet set) = do
     -- We can now attempt to produce a self-contained manifest index from the
     -- collected manifests. If this fails then the package set is not
     -- self-contained.
-    Tuple unsatisfied _ = ManifestIndex.maximalIndex (Set.fromFoldable success)
+    Tuple unsatisfied _ = ManifestIndex.maximalIndex ManifestIndex.IgnoreRanges (Set.fromFoldable success)
 
   -- Otherwise, we can check if we were able to produce an index from the
   -- package set alone, without errors.

app/src/App/Effect/Registry.purs

@@ -252,7 +252,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<
     let formatted = formatPackageVersion name version
     Log.info $ "Writing manifest for " <> formatted <> ":\n" <> printJson Manifest.codec manifest
     index <- Except.rethrow =<< handle env (ReadAllManifests identity)
-    case ManifestIndex.insert manifest index of
+    case ManifestIndex.insert ManifestIndex.ConsiderRanges manifest index of
       Left error ->
         Except.throw $ Array.fold
           [ "Can't insert " <> formatted <> " into manifest index because it has unsatisfied dependencies:"
@@ -275,7 +275,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<
     let formatted = formatPackageVersion name version
     Log.info $ "Deleting manifest for " <> formatted
     index <- Except.rethrow =<< handle env (ReadAllManifests identity)
-    case ManifestIndex.delete name version index of
+    case ManifestIndex.delete ManifestIndex.ConsiderRanges name version index of
       Left error ->
         Except.throw $ Array.fold
           [ "Can't delete " <> formatted <> " from manifest index because it would produce unsatisfied dependencies:"
@@ -359,7 +359,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<
 
         Just metadata -> do
           Log.debug $ "Successfully read metadata for " <> printedName <> " from path " <> path
-          Log.debug $ "Setting metadata cache to singleton entry (as cache was previosuly empty)."
+          Log.debug $ "Setting metadata cache to singleton entry (as cache was previously empty)."
           Cache.put _registryCache AllMetadata (Map.singleton name metadata)
           pure $ Just metadata
 
@@ -836,8 +836,9 @@ readManifestIndexFromDisk root = do
 
   entries <- map partitionEithers $ for packages.success (ManifestIndex.readEntryFile root)
   case entries.fail of
-    [] -> case ManifestIndex.fromSet $ Set.fromFoldable $ Array.foldMap NonEmptyArray.toArray entries.success of
+    [] -> case ManifestIndex.fromSet ManifestIndex.ConsiderRanges $ Set.fromFoldable $ Array.foldMap NonEmptyArray.toArray entries.success of
       Left errors -> do
+        Log.debug $ "Could not read a valid manifest index from entry files: " <> Array.foldMap (Array.foldMap (\(Manifest { name, version }) -> "\n  - " <> formatPackageVersion name version) <<< NonEmptyArray.toArray) entries.success
         Except.throw $ append "Unable to read manifest index (some packages are not satisfiable): " $ Array.foldMap (append "\n  - ") do
           Tuple name versions <- Map.toUnfoldable errors
           Tuple version dependency <- Map.toUnfoldable versions
@@ -878,10 +879,10 @@ readAllMetadataFromDisk metadataDir = do
 
   entries <- Run.liftAff $ map partitionEithers $ for packages.success \name -> do
     result <- readJsonFile Metadata.codec (Path.concat [ metadataDir, PackageName.print name <> ".json" ])
-    pure $ map (Tuple name) result
+    pure $ bimap (Tuple name) (Tuple name) result
 
   unless (Array.null entries.fail) do
-    Except.throw $ append "Could not read metadata for all packages because the metadata directory is invalid (some package metadata cannot be decoded):" $ Array.foldMap (append "\n  - ") entries.fail
+    Except.throw $ append "Could not read metadata for all packages because the metadata directory is invalid (some package metadata cannot be decoded):" $ Array.foldMap (\(Tuple name err) -> "\n  - " <> PackageName.print name <> ": " <> err) entries.fail
 
   Log.debug "Successfully read metadata entries."
   pure $ Map.fromFoldable entries.success