Move ref from metadata to manifest

SPEC.md

@@ -197,6 +197,7 @@ All packages in the registry contain a `purs.json` manifest file in their root d
 - `version`: a valid [`Version`](#version)
 - `license`: a valid [`License`](#license)
 - `location`: a valid [`Location`](#location)
+- `ref`: a `string` representing the reference (e.g., a Git commit or Git tag) at the `location` that was used to fetch this version's source code
 - `owners` (optional): a non-empty array of [`Owner`](#owner)
 - `description` (optional): a description of your library as a plain text string, not markdown, up to 300 characters
 - `includeFiles` (optional): a non-empty array of globs, where globs are used to match file paths (in addition to the `src` directory and other [always-included files](#always-included-files)) that you want included in your package tarball
@@ -221,6 +222,7 @@ For example:
     "githubOwner": "purescript",
     "githubRepo": "purescript-control"
   },
+  "ref": "v4.2.0",
   "include": ["test/**/*.purs"],
   "exclude": ["test/graphs"],
   "dependencies": { "newtype": ">=3.0.0 <4.0.0", "prelude": ">=4.0.0 <5.0.0" }

app/fixtures/registry-index/pr/el/prelude

@@ -1 +1 @@
-{"name":"prelude","version":"6.0.1","license":"BSD-3-Clause","location":{"githubOwner":"purescript","githubRepo":"purescript-prelude"},"description":"The PureScript Prelude","dependencies":{}}
+{"name":"prelude","version":"6.0.1","license":"BSD-3-Clause","location":{"githubOwner":"purescript","githubRepo":"purescript-prelude"},"ref":"v6.0.1","description":"The PureScript Prelude","dependencies":{}}

app/fixtures/registry-index/ty/pe/type-equality

@@ -1 +1 @@
-{"name":"type-equality","version":"4.0.1","license":"BSD-3-Clause","location":{"githubOwner":"purescript","githubRepo":"purescript-type-equality"},"dependencies":{}}
+{"name":"type-equality","version":"4.0.1","license":"BSD-3-Clause","location":{"githubOwner":"purescript","githubRepo":"purescript-type-equality"},"ref":"v4.0.1","dependencies":{}}

app/src/App/API.purs

@@ -481,7 +481,7 @@ publish maybeLegacyIndex payload = do
       Log.notice $ "Package source does not have a purs.json file, creating one from your spago.yaml file..."
       SpagoYaml.readSpagoYaml packageSpagoYaml >>= case _ of
         Left readErr -> Except.throw $ "Could not publish your package - a spago.yaml was present, but it was not possible to read it:\n" <> readErr
-        Right config -> case SpagoYaml.spagoYamlToManifest config of
+        Right config -> case SpagoYaml.spagoYamlToManifest payload.ref config of
           Left err -> Except.throw $ "Could not publish your package - there was an error while converting your spago.yaml into a purs.json manifest:\n" <> err
           Right manifest -> do
             Log.notice $ Array.fold
@@ -508,7 +508,7 @@ publish maybeLegacyIndex payload = do
             ]
         Right legacyManifest -> do
           Log.debug $ "Successfully produced a legacy manifest from the package source."
-          let manifest = Legacy.Manifest.toManifest payload.name version existingMetadata.location legacyManifest
+          let manifest = Legacy.Manifest.toManifest payload.name version existingMetadata.location payload.ref legacyManifest
           Log.notice $ Array.fold
             [ "Converted your legacy manifest(s) into a purs.json manifest to use for publishing:"
             , "\n```json\n"
@@ -774,7 +774,7 @@ publish maybeLegacyIndex payload = do
 
       Storage.upload (un Manifest manifest).name (un Manifest manifest).version tarballPath
       Log.debug $ "Adding the new version " <> Version.print (un Manifest manifest).version <> " to the package metadata file."
-      let newPublishedVersion = { hash, ref: payload.ref, compilers: NonEmptyArray.singleton payload.compiler, publishedTime, bytes }
+      let newPublishedVersion = { hash, compilers: NonEmptyArray.singleton payload.compiler, publishedTime, bytes }
       let newMetadata = metadata { published = Map.insert (un Manifest manifest).version newPublishedVersion metadata.published }
 
       Registry.writeMetadata (un Manifest manifest).name (Metadata newMetadata)

app/src/App/Effect/Registry.purs

@@ -502,10 +502,9 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<
     Log.info $ "Mirroring legacy package set " <> name <> " to the legacy package sets repo"
 
     manifests <- Except.rethrow =<< handle env (ReadAllManifests identity)
-    metadata <- Except.rethrow =<< handle env (ReadAllMetadata identity)
 
     Log.debug $ "Converting package set..."
-    converted <- case Legacy.PackageSet.convertPackageSet manifests metadata set of
+    converted <- case Legacy.PackageSet.convertPackageSet manifests set of
       Left error -> Except.throw $ "Failed to convert package set " <> name <> " to a legacy package set: " <> error
       Right converted -> pure converted
 

app/src/App/Legacy/Manifest.purs

@@ -59,13 +59,13 @@ type LegacyManifest =
   , dependencies :: Map PackageName Range
   }
 
-toManifest :: PackageName -> Version -> Location -> LegacyManifest -> Manifest
-toManifest name version location legacy = do
+toManifest :: PackageName -> Version -> Location -> String -> LegacyManifest -> Manifest
+toManifest name version location ref legacy = do
   let { license, description, dependencies } = patchLegacyManifest name version legacy
   let includeFiles = Nothing
   let excludeFiles = Nothing
   let owners = Nothing
-  Manifest { name, version, location, license, description, dependencies, includeFiles, excludeFiles, owners }
+  Manifest { name, version, location, ref, license, description, dependencies, includeFiles, excludeFiles, owners }
 
 -- | Attempt to retrieve a license, description, and set of dependencies from a
 -- | PureScript repo that does not have a Registry-supported manifest, but does

app/src/App/Legacy/PackageSet.purs

@@ -102,8 +102,8 @@ printPscTag (PscTag { compiler, date }) =
     , Format.DateTime.format pscDateFormat (DateTime date bottom)
     ]
 
-convertPackageSet :: ManifestIndex -> Map PackageName Metadata -> PackageSet -> Either String ConvertedLegacyPackageSet
-convertPackageSet index metadataMap (PackageSet { compiler, packages, published, version }) = do
+convertPackageSet :: ManifestIndex -> PackageSet -> Either String ConvertedLegacyPackageSet
+convertPackageSet index (PackageSet { compiler, packages, published, version }) = do
   converted <- case separate $ mapWithIndex convertPackage packages of
     { left, right } | Map.isEmpty left -> Right right
     { left } -> do
@@ -130,17 +130,14 @@ convertPackageSet index metadataMap (PackageSet { compiler, packages, published,
     versions <- note noIndexPackageError $ Map.lookup packageName $ ManifestIndex.toMap index
     Manifest manifest <- note noIndexVersionError $ Map.lookup packageVersion versions
 
-    Metadata metadata <- note noMetadataPackageError $ Map.lookup packageName metadataMap
-    { ref } <- note noMetadataVersionError $ Map.lookup packageVersion metadata.published
-
-    repo <- case metadata.location of
+    repo <- case manifest.location of
       GitHub { owner, repo, subdir: Nothing } -> Right $ "https://github.com/" <> owner <> "/" <> repo <> ".git"
       Git { url, subdir: Nothing } -> Right url
       GitHub _ -> Left usesSubdirError
       Git _ -> Left usesSubdirError
 
     pure
-      { version: RawVersion ref
+      { version: RawVersion manifest.ref
       , dependencies: Array.fromFoldable $ Map.keys $ manifest.dependencies
       , repo
       }
@@ -149,8 +146,6 @@ convertPackageSet index metadataMap (PackageSet { compiler, packages, published,
     versionStr = Version.print packageVersion
     noIndexPackageError = "No registry index entry found for " <> nameStr
     noIndexVersionError = "Found registry index entry for " <> nameStr <> " but none for version " <> versionStr
-    noMetadataPackageError = "No metadata entry found for " <> nameStr
-    noMetadataVersionError = "Found metadata entry for " <> nameStr <> " but no published version for " <> versionStr
     usesSubdirError = "Package " <> nameStr <> " uses the 'subdir' key, which is not supported for legacy package sets."
 
 printDhall :: LegacyPackageSet -> String

app/src/App/Manifest/SpagoYaml.purs

@@ -27,9 +27,10 @@ import Registry.Range (Range)
 import Registry.Range as Range
 import Registry.Version as Version
 
--- | Attempt to convert a spago.yaml file to a Manifest
-spagoYamlToManifest :: SpagoYaml -> Either String Manifest
-spagoYamlToManifest config = do
+-- | Attempt to convert a spago.yaml file to a Manifest. The ref parameter is
+-- | the Git reference (tag or commit) used to fetch this version's source.
+spagoYamlToManifest :: String -> SpagoYaml -> Either String Manifest
+spagoYamlToManifest ref config = do
   package@{ name, description, dependencies: spagoDependencies } <- note "No 'package' key found in config." config.package
   publish@{ version, license, owners } <- note "No 'publish' key found under the 'package' key in config." package.publish
   location <- note "No 'location' key found under the 'publish' key in config." publish.location
@@ -43,6 +44,7 @@ spagoYamlToManifest config = do
     , description
     , license
     , location
+    , ref
     , owners
     , includeFiles
     , excludeFiles

app/test/App/Legacy/PackageSet.purs

@@ -2,8 +2,6 @@ module Test.Registry.App.Legacy.PackageSet (spec) where
 
 import Registry.App.Prelude
 
-import Data.Array.NonEmpty as NonEmptyArray
-import Data.DateTime (DateTime(..))
 import Data.Either as Either
 import Data.Map as Map
 import Data.Set as Set
@@ -14,7 +12,6 @@ import Registry.App.Legacy.PackageSet as Legacy.PackageSet
 import Registry.App.Legacy.Types (legacyPackageSetCodec)
 import Registry.ManifestIndex as ManifestIndex
 import Registry.PackageName as PackageName
-import Registry.Sha256 as Sha256
 import Registry.Test.Assert as Assert
 import Registry.Test.Utils as Utils
 import Registry.Version as Version
@@ -93,7 +90,7 @@ packageSet = PackageSet
 
 convertedPackageSet :: ConvertedLegacyPackageSet
 convertedPackageSet =
-  case Legacy.PackageSet.convertPackageSet index metadata packageSet of
+  case Legacy.PackageSet.convertPackageSet index packageSet of
     Left err -> unsafeCrashWith err
     Right value -> value
   where
@@ -104,13 +101,6 @@ convertedPackageSet =
     , mkManifest prelude []
     ]
 
-  metadata = Map.fromFoldable
-    [ unsafeMetadataEntry assert
-    , unsafeMetadataEntry console
-    , unsafeMetadataEntry effect
-    , unsafeMetadataEntry prelude
-    ]
-
 legacyPackageSetJson :: String
 legacyPackageSetJson =
   """{
@@ -201,23 +191,3 @@ mkManifest (Tuple name version) deps = do
     (PackageName.print name)
     (LenientVersion.print version)
     (map (bimap PackageName.print (LenientVersion.version >>> toRange)) deps)
-
-unsafeMetadataEntry :: Tuple PackageName LenientVersion -> Tuple PackageName Metadata
-unsafeMetadataEntry (Tuple name version) = do
-  let
-    published =
-      { ref: LenientVersion.raw version
-      , hash: unsafeFromRight $ Sha256.parse "sha256-gb24ZRec6mgR8TFBVR2eIh5vsMdhuL+zK9VKjWP74Cw="
-      , bytes: 0.0
-      , compilers: NonEmptyArray.singleton (Utils.unsafeVersion "0.15.2")
-      , publishedTime: DateTime (Utils.unsafeDate "2022-07-07") bottom
-      }
-
-    metadata = Metadata
-      { location: GitHub { owner: "purescript", repo: "purescript-" <> PackageName.print name, subdir: Nothing }
-      , owners: Nothing
-      , published: Map.singleton (LenientVersion.version version) published
-      , unpublished: Map.empty
-      }
-
-  Tuple name metadata

app/test/App/Manifest/SpagoYaml.purs

@@ -19,6 +19,6 @@ spec = do
       config <- SpagoYaml.readSpagoYaml (Path.concat [ fixturesPath, path ]) >>= case _ of
         Left err -> Aff.throwError $ Aff.error err
         Right config -> pure config
-      case SpagoYaml.spagoYamlToManifest config of
+      case SpagoYaml.spagoYamlToManifest "v1.0.0" config of
         Left err -> Assert.fail $ path <> " failed: " <> err
         Right _ -> pure unit

lib/fixtures/manifests/aff-5.1.2.json

@@ -6,6 +6,7 @@
     "githubOwner": "purescript",
     "githubRepo": "purescript-aff"
   },
+  "ref": "v5.1.2",
   "dependencies": {
     "datetime": ">=4.0.0 <5.0.0",
     "effect": ">=2.0.0 <3.0.0",

lib/fixtures/manifests/mysql-4.1.1.json

@@ -6,6 +6,7 @@
     "githubOwner": "oreshinya",
     "githubRepo": "purescript-mysql"
   },
+  "ref": "v4.1.1",
   "dependencies": {
     "aff": ">=5.0.2 <6.0.0",
     "js-date": ">=6.0.0 <7.0.0",

lib/fixtures/manifests/prelude-4.1.1.json

@@ -7,6 +7,7 @@
     "githubOwner": "purescript",
     "githubRepo": "purescript-prelude"
   },
+  "ref": "v4.1.1",
   "owners": [
     {
       "keytype": "ed-25519",

lib/src/Manifest.purs

@@ -48,6 +48,7 @@ newtype Manifest = Manifest
   , version :: Version
   , license :: License
   , location :: Location
+  , ref :: String
   , owners :: Maybe (NonEmptyArray Owner)
   , description :: Maybe String
   , includeFiles :: Maybe (NonEmptyArray NonEmptyString)
@@ -77,6 +78,7 @@ codec = Profunctor.wrapIso Manifest $ CJ.named "Manifest" $ CJ.object
   $ CJ.recordProp @"license" License.codec
   $ CJ.recordPropOptional @"description" (Internal.Codec.limitedString 300)
   $ CJ.recordProp @"location" Location.codec
+  $ CJ.recordProp @"ref" CJ.string
   $ CJ.recordPropOptional @"owners" (CJ.Common.nonEmptyArray Owner.codec)
   $ CJ.recordPropOptional @"includeFiles" (CJ.Common.nonEmptyArray CJ.Common.nonEmptyString)
   $ CJ.recordPropOptional @"excludeFiles" (CJ.Common.nonEmptyArray CJ.Common.nonEmptyString)

lib/src/Metadata.purs

@@ -63,17 +63,11 @@ codec = Profunctor.wrapIso Metadata $ CJ.named "Metadata" $ CJ.object
   $ CJ.record
 
 -- | Metadata about a published package version.
--- |
--- | NOTE: The `ref` field is UNSPECIFIED and WILL BE REMOVED in the future. Do
--- | not rely on its presence!
 type PublishedMetadata =
   { bytes :: Number
   , compilers :: NonEmptyArray Version
   , hash :: Sha256
   , publishedTime :: DateTime
-
-  -- UNSPECIFIED: Will be removed in the future.
-  , ref :: String
   }
 
 publishedMetadataCodec :: CJ.Codec PublishedMetadata
@@ -82,7 +76,6 @@ publishedMetadataCodec = CJ.named "PublishedMetadata" $ CJ.Record.object
   , compilers: CJ.Common.nonEmptyArray Version.codec
   , hash: Sha256.codec
   , publishedTime: Internal.Codec.iso8601DateTime
-  , ref: CJ.string
   }
 
 -- | Metadata about an unpublished package version.

lib/test/Registry/ManifestIndex.purs

@@ -151,9 +151,9 @@ spec = do
 
 contextEntry :: String
 contextEntry =
-  """{"name":"context","version":"0.0.1","license":"MIT","location":{"githubOwner":"Fresheyeball","githubRepo":"purescript-owner"},"dependencies":{}}
-{"name":"context","version":"0.0.2","license":"MIT","location":{"githubOwner":"Fresheyeball","githubRepo":"purescript-owner"},"dependencies":{}}
-{"name":"context","version":"0.0.3","license":"MIT","location":{"githubOwner":"Fresheyeball","githubRepo":"purescript-owner"},"dependencies":{}}
+  """{"name":"context","version":"0.0.1","license":"MIT","location":{"githubOwner":"Fresheyeball","githubRepo":"purescript-owner"},"ref":"v0.0.1","dependencies":{}}
+{"name":"context","version":"0.0.2","license":"MIT","location":{"githubOwner":"Fresheyeball","githubRepo":"purescript-owner"},"ref":"v0.0.2","dependencies":{}}
+{"name":"context","version":"0.0.3","license":"MIT","location":{"githubOwner":"Fresheyeball","githubRepo":"purescript-owner"},"ref":"v0.0.3","dependencies":{}}
 """
 
 testIndex
@@ -242,6 +242,7 @@ manifestCodec' = Profunctor.dimap to from $ CJ.named "ManifestRep" $ CJ.Record.o
         { url: "https://github.com/purescript/purescript-" <> PackageName.print name <> ".git"
         , subdir: Nothing
         }
+    , ref: "v" <> Version.print version
     , description: Nothing
     , owners: Nothing
     , includeFiles: Nothing

lib/test/Registry/Metadata.purs

@@ -29,26 +29,23 @@ recordStudio =
         "0.13.0"
       ],
       "hash": "sha256-LPRUC8ozZc7VCeRhKa4CtSgAfNqgAoVs2lH+7mYEcTk=",
-      "publishedTime": "2021-03-27T10:03:46.000Z",
-      "ref": "v0.1.0"
+      "publishedTime": "2021-03-27T10:03:46.000Z"
     },
     "0.2.1": {
       "bytes": 3365,
       "compilers": [
         "0.13.0"
       ],
       "hash": "sha256-ySKKKp3rUJa4UmYTZshaOMO3jE+DW7IIqKJsurA2PP8=",
-      "publishedTime": "2022-05-15T10:51:57.000Z",
-      "ref": "v0.2.1"
+      "publishedTime": "2022-05-15T10:51:57.000Z"
     },
     "1.0.0": {
       "bytes": 5155,
       "compilers": [
         "0.13.0"
       ],
       "hash": "sha256-0iMF8Rq88QBGuxTNrh+iuruw8l5boCP6J2JWBpQ4b7w=",
-      "publishedTime": "2022-11-03T17:30:28.000Z",
-      "ref": "v1.0.0"
+      "publishedTime": "2022-11-03T17:30:28.000Z"
     },
     "1.0.1": {
       "bytes": 5635,
@@ -57,8 +54,7 @@ recordStudio =
         "0.13.1"
       ],
       "hash": "sha256-Xm9pwDBHW5zYUEzxfVSgjglIcwRI1gcCOmcpyQ/tqeY=",
-      "publishedTime": "2022-11-04T12:21:09.000Z",
-      "ref": "v1.0.1"
+      "publishedTime": "2022-11-04T12:21:09.000Z"
     }
   },
   "unpublished": {

lib/test/Registry/Operation/Validation.purs

@@ -67,7 +67,7 @@ spec = do
       inRange = unsafeDateTime "2022-12-11T12:00:00.000Z"
       compilers = NonEmptyArray.singleton (unsafeVersion "0.13.0")
 
-      publishedMetadata = { bytes: 100.0, hash: defaultHash, publishedTime: outOfRange, compilers, ref: "" }
+      publishedMetadata = { bytes: 100.0, hash: defaultHash, publishedTime: outOfRange, compilers }
 
       metadata = Metadata
         { location: defaultLocation

scripts/src/LegacyImporter.purs

@@ -761,7 +761,7 @@ buildLegacyPackageManifests rawPackage rawUrl = Run.Except.runExceptAt _exceptPa
                     Legacy.Manifest.fetchLegacyManifest package.name package.address (RawVersion tag.name) >>= case _ of
                       Left error -> throwVersion { error: InvalidManifest error, reason: "Legacy manifest could not be parsed." }
                       Right result -> pure result
-                  pure $ Legacy.Manifest.toManifest package.name (LenientVersion.version version) location legacyManifest
+                  pure $ Legacy.Manifest.toManifest package.name (LenientVersion.version version) location tag.name legacyManifest
                 case manifest of
                   Left err -> Log.info $ "Failed to build manifest for " <> PackageName.print package.name <> "@" <> tag.name <> ": " <> printJson versionValidationErrorCodec err
                   Right val -> Log.info $ "Built manifest for " <> PackageName.print package.name <> "@" <> tag.name <> ":\n" <> printJson Manifest.codec val
@@ -1463,7 +1463,7 @@ fetchSpagoYaml address ref = do
           | location /= GitHub { owner: address.owner, repo: address.repo, subdir: Nothing } -> do
               Log.warn "spago.yaml file does not use the same location it was fetched from, this is disallowed..."
               pure Nothing
-        Right config -> case SpagoYaml.spagoYamlToManifest config of
+        Right config -> case SpagoYaml.spagoYamlToManifest (un RawVersion ref) config of
           Left err -> do
             Log.warn $ "Failed to convert parsed spago.yaml file to purs.json " <> contents <> "\nwith errors:\n" <> err
             pure Nothing

scripts/src/PackageDeleter.purs

@@ -228,21 +228,24 @@ deleteVersion arguments name version = do
           Just published, Nothing -> pure (Just (Right published))
           Nothing, Just unpublished -> pure (Just (Left unpublished))
           Nothing, Nothing -> pure Nothing
+      -- Read manifest before deleting it (needed for reimport)
+      maybeManifest <- Registry.readManifest name version
       let
         newMetadata = Metadata $ oldMetadata { published = Map.delete version oldMetadata.published, unpublished = Map.delete version oldMetadata.unpublished }
       Registry.writeMetadata name newMetadata
       Registry.deleteManifest name version
       -- --reimport
       when arguments.reimport do
-        case publishment of
-          Nothing -> Log.error "Cannot reimport a version that was not published"
-          Just (Left _) -> Log.error "Cannot reimport a version that was specifically unpublished"
-          Just (Right specificPackageMetadata) -> do
+        case publishment, maybeManifest of
+          Nothing, _ -> Log.error "Cannot reimport a version that was not published"
+          Just (Left _), _ -> Log.error "Cannot reimport a version that was specifically unpublished"
+          Just (Right _), Nothing -> Log.error $ "Cannot reimport: manifest not found for " <> formatted
+          Just (Right _), Just (Manifest manifest) -> do
             -- Obtains `newMetadata` via cache
             void $ API.publish Nothing
               { location: Just oldMetadata.location
               , name: name
-              , ref: specificPackageMetadata.ref
+              , ref: manifest.ref
               , version: version
               , compiler: unsafeFromRight $ Version.parse "0.15.4"
               , resolutions: Nothing