From 63b6c30c899f8014ec615ab7d4c2ea04acd70ae0 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Wed, 2 Oct 2024 17:02:02 -0700
Subject: [PATCH 01/14] - release cli command stubs - adding docker registry
 image

---
 docker-compose.yml          |  6 ++++++
 frontend/cli/cmd_release.go | 32 ++++++++++++++++++++++++++++++++
 frontend/cli/main.go        |  1 +
 3 files changed, 39 insertions(+)
 create mode 100644 frontend/cli/cmd_release.go

diff --git a/docker-compose.yml b/docker-compose.yml
index 51b3c7b0ac..e509d23e71 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -53,6 +53,12 @@ services:
     environment:
       SERVICES: secretsmanager
       DEBUG: 1
+  registry:
+    image: registry:2
+    ports:
+      -  "5000:5000"
+    volumes:
+      - ./.registry:/var/lib/registry
 
 volumes:
   grafana-storage: {}
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
new file mode 100644
index 0000000000..5e9f17642a
--- /dev/null
+++ b/frontend/cli/cmd_release.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+)
+
+type releaseCmd struct {
+	Describe releaseDescribeCmd `cmd:"" help:"Describes the specified release."`
+	Publish  releasePublishCmd  `cmd:"" help:"Packages the project into a release and publishes it."`
+	List     releaseListCmd     `cmd:"" help:"Lists all published releases."`
+}
+
+type releaseDescribeCmd struct {
+}
+
+func (d *releaseDescribeCmd) Run() error {
+	return fmt.Errorf("release describe not implemented")
+}
+
+type releasePublishCmd struct {
+}
+
+func (d *releasePublishCmd) Run() error {
+	return fmt.Errorf("release publish not implemented")
+}
+
+type releaseListCmd struct {
+}
+
+func (d *releaseListCmd) Run() error {
+	return fmt.Errorf("release list not implemented")
+}
diff --git a/frontend/cli/main.go b/frontend/cli/main.go
index 58c61f336a..ebb69671b4 100644
--- a/frontend/cli/main.go
+++ b/frontend/cli/main.go
@@ -54,6 +54,7 @@ type InteractiveCLI struct {
 	Secret   secretCmd   `cmd:"" help:"Manage secrets."`
 	Config   configCmd   `cmd:"" help:"Manage configuration."`
 	Pubsub   pubsubCmd   `cmd:"" help:"Manage pub/sub."`
+	Release  releaseCmd  `cmd:"" help:"Manage releases."`
 }
 
 type CLI struct {

From e7a480df6cb2178cf705a4f4075243f88066338a Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Wed, 2 Oct 2024 17:02:02 -0700
Subject: [PATCH 02/14] - release cli command stubs - adding docker registry
 image

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 3bd8e3b820..7b77684553 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .hermit/
 .vscode/*
+.registry/
 !/.vscode/settings.json
 !/.vscode/launch.json
 .idea/*

From 02d4279ce8a3ff372e26b28294c2c3f5b02565ad Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Thu, 3 Oct 2024 17:08:18 -0700
Subject: [PATCH 03/14] - partial oci registry support - integration with CLI
 to demonstrate release publishing is WIP

---
 backend/controller/artefacts/dal_registry.go |  6 +-
 backend/controller/artefacts/oci_registry.go | 96 ++++++++++++++++++++
 frontend/cli/cmd_release.go                  | 34 ++++++-
 go.mod                                       |  1 +
 go.sum                                       |  2 +
 5 files changed, 137 insertions(+), 2 deletions(-)
 create mode 100644 backend/controller/artefacts/oci_registry.go

diff --git a/backend/controller/artefacts/dal_registry.go b/backend/controller/artefacts/dal_registry.go
index 42e7e9728e..6a1442a66d 100644
--- a/backend/controller/artefacts/dal_registry.go
+++ b/backend/controller/artefacts/dal_registry.go
@@ -76,7 +76,11 @@ func (s *Service) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCl
 }
 
 func (s *Service) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) {
-	rows, err := s.db.GetDeploymentArtefacts(ctx, releaseID)
+	return getDatabaseReleaseArtefacts(ctx, s.db, releaseID)
+}
+
+func getDatabaseReleaseArtefacts(ctx context.Context, db sql.Querier, releaseID int64) ([]ReleaseArtefact, error) {
+	rows, err := db.GetDeploymentArtefacts(ctx, releaseID)
 	if err != nil {
 		return nil, fmt.Errorf("unable to get release artefacts: %w", libdal.TranslatePGError(err))
 	}
diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
new file mode 100644
index 0000000000..81af0970d0
--- /dev/null
+++ b/backend/controller/artefacts/oci_registry.go
@@ -0,0 +1,96 @@
+package artefacts
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
+	"github.com/TBD54566975/ftl/backend/libdal"
+	"github.com/TBD54566975/ftl/internal/model"
+	"github.com/TBD54566975/ftl/internal/sha256"
+	"io"
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/registry/remote"
+	"oras.land/oras-go/v2/registry/remote/auth"
+	"oras.land/oras-go/v2/registry/remote/retry"
+)
+
+type ContainerConfig struct {
+	Registry   string
+	Repository string
+	Username   string
+	Password   string
+}
+
+type ContainerService struct {
+	host       string
+	repository *remote.Repository
+	// in the interim releases and artefacts will continue to be linked via the `deployment_artefacts` table
+	Handle *libdal.Handle[ContainerService]
+	db     sql.Querier
+}
+
+func NewContainerService(c ContainerConfig, conn libdal.Connection) *ContainerService {
+	repository, err := remote.NewRepository(fmt.Sprintf("%s/%s", c.Registry, c.Repository))
+	if err != nil {
+		panic(fmt.Errorf("unable to connect to OCI repository: %w", err))
+	}
+
+	client := &auth.Client{
+		Client: retry.DefaultClient,
+		Cache:  auth.NewCache(),
+		Credential: auth.StaticCredential(c.Registry, auth.Credential{
+			Username: c.Username,
+			Password: c.Password,
+		}),
+	}
+
+	s := &ContainerService{
+		host:       c.Registry,
+		repository: repository,
+		Handle: libdal.New(conn, func(h *libdal.Handle[ContainerService]) *ContainerService {
+			svc := &ContainerService{
+				host:       c.Registry,
+				repository: repository,
+				Handle:     h,
+				db:         sql.New(h.Connection),
+			}
+			svc.repository.Client = client
+			return svc
+		}),
+	}
+
+	return s
+}
+
+func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.SHA256) (keys []ArtefactKey, missing []sha256.SHA256, err error) {
+	return nil, nil, nil
+}
+
+func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha256.SHA256, error) {
+	_, err := oras.PushBytes(ctx, s.repository, remoteModulePath(s.host, artefact.Digest), artefact.Content)
+	if err != nil {
+		return sha256.SHA256{}, fmt.Errorf("unable to upload artefact: %w", err)
+	}
+	return sha256.SHA256{}, nil
+}
+
+func (s *ContainerService) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCloser, error) {
+	_, stream, err := oras.Fetch(ctx, s.repository, remoteModulePath(s.host, digest), oras.DefaultFetchOptions)
+	if err != nil {
+		return nil, fmt.Errorf("unable to download artefact: %w", err)
+	}
+	return stream, nil
+}
+
+func (s *ContainerService) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) {
+	return nil, nil
+}
+
+func (s *ContainerService) AddReleaseArtefact(ctx context.Context, key model.DeploymentKey, ra ReleaseArtefact) error {
+	return nil
+}
+
+func remoteModulePath(host string, digest sha256.SHA256) string {
+	return fmt.Sprintf("%s/modules/%s:latest", host, hex.EncodeToString(digest[:]))
+}
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index 5e9f17642a..97c1a36921 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -1,7 +1,12 @@
 package main
 
 import (
+	"context"
+	"crypto/sha256"
 	"fmt"
+	"github.com/TBD54566975/ftl/backend/controller/artefacts"
+	internalobservability "github.com/TBD54566975/ftl/internal/observability"
+	"github.com/google/uuid"
 )
 
 type releaseCmd struct {
@@ -11,6 +16,7 @@ type releaseCmd struct {
 }
 
 type releaseDescribeCmd struct {
+	Digest string `arg:"" help:"Digest of the target release."`
 }
 
 func (d *releaseDescribeCmd) Run() error {
@@ -18,10 +24,36 @@ func (d *releaseDescribeCmd) Run() error {
 }
 
 type releasePublishCmd struct {
+	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
+	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
+	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
 }
 
 func (d *releasePublishCmd) Run() error {
-	return fmt.Errorf("release publish not implemented")
+	conn, err := internalobservability.OpenDBAndInstrument(d.DSN)
+	if err != nil {
+		return fmt.Errorf("failed to open DB connection: %w", err)
+	}
+	conn.SetMaxIdleConns(d.MaxIdleDBConnections)
+	conn.SetMaxOpenConns(d.MaxOpenDBConnections)
+
+	svc := artefacts.NewContainerService(artefacts.ContainerConfig{
+		Registry:   "localhost:5000",
+		Repository: "demo-repo",
+	}, conn)
+	content := uuid.New()
+	contentBytes := content[:]
+	_, err = svc.Upload(context.Background(), artefacts.Artefact{
+		Digest: sha256.Sum256(contentBytes),
+		Metadata: artefacts.Metadata{
+			Path: fmt.Sprintf("random/%s", content),
+		},
+		Content: contentBytes,
+	})
+	if err != nil {
+		return fmt.Errorf("failed upload artefact: %w", err)
+	}
+	return nil
 }
 
 type releaseListCmd struct {
diff --git a/go.mod b/go.mod
index d5326bf547..870ac52542 100644
--- a/go.mod
+++ b/go.mod
@@ -153,6 +153,7 @@ require (
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
 	modernc.org/libc v1.55.3 // indirect
+	oras.land/oras-go/v2 v2.5.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
diff --git a/go.sum b/go.sum
index 0bcb692658..82a885f715 100644
--- a/go.sum
+++ b/go.sum
@@ -508,6 +508,8 @@ modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
 modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
 modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c=
+oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=

From 3ae28f823e9f3f4fd59d39070578dc6c84637b96 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Fri, 4 Oct 2024 00:17:17 +0000
Subject: [PATCH 04/14] chore(autofmt): Automated formatting

---
 backend/controller/artefacts/oci_registry.go | 10 ++++++----
 frontend/cli/cmd_release.go                  |  4 +++-
 go.mod                                       |  2 +-
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index 81af0970d0..8d2635a719 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -4,15 +4,17 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
-	"github.com/TBD54566975/ftl/backend/libdal"
-	"github.com/TBD54566975/ftl/internal/model"
-	"github.com/TBD54566975/ftl/internal/sha256"
 	"io"
+
 	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/registry/remote"
 	"oras.land/oras-go/v2/registry/remote/auth"
 	"oras.land/oras-go/v2/registry/remote/retry"
+
+	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
+	"github.com/TBD54566975/ftl/backend/libdal"
+	"github.com/TBD54566975/ftl/internal/model"
+	"github.com/TBD54566975/ftl/internal/sha256"
 )
 
 type ContainerConfig struct {
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index 97c1a36921..4754188759 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -4,9 +4,11 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
+
+	"github.com/google/uuid"
+
 	"github.com/TBD54566975/ftl/backend/controller/artefacts"
 	internalobservability "github.com/TBD54566975/ftl/internal/observability"
-	"github.com/google/uuid"
 )
 
 type releaseCmd struct {
diff --git a/go.mod b/go.mod
index 870ac52542..eb54c3a82e 100644
--- a/go.mod
+++ b/go.mod
@@ -80,6 +80,7 @@ require (
 	k8s.io/apimachinery v0.31.1
 	k8s.io/client-go v0.31.1
 	modernc.org/sqlite v1.33.1
+	oras.land/oras-go/v2 v2.5.0
 )
 
 require (
@@ -153,7 +154,6 @@ require (
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
 	modernc.org/libc v1.55.3 // indirect
-	oras.land/oras-go/v2 v2.5.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect

From 66552ff4dcfb884734a0d617265f3a16b3a884d6 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Fri, 4 Oct 2024 14:10:46 -0700
Subject: [PATCH 05/14] publishing artifacts from a memory to registry

---
 backend/controller/artefacts/oci_registry.go | 118 +++++++++++++------
 frontend/cli/cmd_release.go                  |   4 +-
 2 files changed, 81 insertions(+), 41 deletions(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index 8d2635a719..9c05a38da3 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -1,68 +1,74 @@
 package artefacts
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"fmt"
+	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
+	"github.com/TBD54566975/ftl/backend/libdal"
+	"github.com/TBD54566975/ftl/internal/model"
+	"github.com/TBD54566975/ftl/internal/sha256"
+	"github.com/opencontainers/go-digest"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"io"
-
 	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/content/memory"
 	"oras.land/oras-go/v2/registry/remote"
 	"oras.land/oras-go/v2/registry/remote/auth"
 	"oras.land/oras-go/v2/registry/remote/retry"
-
-	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
-	"github.com/TBD54566975/ftl/backend/libdal"
-	"github.com/TBD54566975/ftl/internal/model"
-	"github.com/TBD54566975/ftl/internal/sha256"
 )
 
 type ContainerConfig struct {
-	Registry   string
-	Repository string
-	Username   string
-	Password   string
+	Registry       string
+	Username       string
+	Password       string
+	AllowPlainHTTP bool
 }
 
 type ContainerService struct {
-	host       string
-	repository *remote.Repository
+	host              string
+	connectionBuilder func(container string) (*remote.Repository, error)
+
 	// in the interim releases and artefacts will continue to be linked via the `deployment_artefacts` table
 	Handle *libdal.Handle[ContainerService]
 	db     sql.Querier
 }
 
 func NewContainerService(c ContainerConfig, conn libdal.Connection) *ContainerService {
-	repository, err := remote.NewRepository(fmt.Sprintf("%s/%s", c.Registry, c.Repository))
-	if err != nil {
-		panic(fmt.Errorf("unable to connect to OCI repository: %w", err))
-	}
+	// Connect the registry targeting the specified container
+	connectionBuilder := func(container string) (*remote.Repository, error) {
+		ref := fmt.Sprintf("%s/%s", c.Registry, container)
+		reg, err := remote.NewRepository(ref)
+		if err != nil {
+			return nil, fmt.Errorf("unable to connect to container registry '%s': %w", ref, err)
+		}
 
-	client := &auth.Client{
-		Client: retry.DefaultClient,
-		Cache:  auth.NewCache(),
-		Credential: auth.StaticCredential(c.Registry, auth.Credential{
-			Username: c.Username,
-			Password: c.Password,
-		}),
+		reg.Client = &auth.Client{
+			Client: retry.DefaultClient,
+			Cache:  auth.NewCache(),
+			Credential: auth.StaticCredential(c.Registry, auth.Credential{
+				Username: c.Username,
+				Password: c.Password,
+			}),
+		}
+		reg.PlainHTTP = c.AllowPlainHTTP
+
+		return reg, nil
 	}
 
-	s := &ContainerService{
-		host:       c.Registry,
-		repository: repository,
+	return &ContainerService{
+		host:              c.Registry,
+		connectionBuilder: connectionBuilder,
 		Handle: libdal.New(conn, func(h *libdal.Handle[ContainerService]) *ContainerService {
-			svc := &ContainerService{
-				host:       c.Registry,
-				repository: repository,
-				Handle:     h,
-				db:         sql.New(h.Connection),
+			return &ContainerService{
+				host:              c.Registry,
+				connectionBuilder: connectionBuilder,
+				Handle:            h,
+				db:                sql.New(h.Connection),
 			}
-			svc.repository.Client = client
-			return svc
 		}),
 	}
-
-	return s
 }
 
 func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.SHA256) (keys []ArtefactKey, missing []sha256.SHA256, err error) {
@@ -70,15 +76,49 @@ func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.
 }
 
 func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha256.SHA256, error) {
-	_, err := oras.PushBytes(ctx, s.repository, remoteModulePath(s.host, artefact.Digest), artefact.Content)
+	hash := sha256.Sum(artefact.Content)
+	ref := fmt.Sprintf("ftl/modules/%s", hash)
+	ms := memory.New()
+	mediaDescriptor := v1.Descriptor{
+		MediaType: "application/ftl.module.v1",
+		Digest:    digest.NewDigestFromBytes(digest.SHA256, hash[:]),
+		Size:      int64(len(artefact.Content)),
+	}
+	err := ms.Push(ctx, mediaDescriptor, bytes.NewReader(artefact.Content))
 	if err != nil {
-		return sha256.SHA256{}, fmt.Errorf("unable to upload artefact: %w", err)
+		return sha256.SHA256{}, fmt.Errorf("unable to stage artefact in memory: %w", err)
+	}
+	artifactType := "application/ftl.module.artifact"
+	opts := oras.PackManifestOptions{
+		Layers: []v1.Descriptor{mediaDescriptor},
+	}
+	tag := "latest"
+	manifestDescriptor, err := oras.PackManifest(ctx, ms, oras.PackManifestVersion1_1, artifactType, opts)
+	if err != nil {
+		return sha256.SHA256{}, fmt.Errorf("unable to pack artifact manifest: %w", err)
+	}
+	if err = ms.Tag(ctx, manifestDescriptor, tag); err != nil {
+		return sha256.SHA256{}, fmt.Errorf("unable to tag artifact: %w", err)
 	}
-	return sha256.SHA256{}, nil
+	registry, err := s.connectionBuilder(ref)
+	if err != nil {
+		return sha256.SHA256{}, fmt.Errorf("unable to connect to registry '%s/%s': %w", s.host, ref, err)
+	}
+	desc, err := oras.Copy(ctx, ms, tag, registry, tag, oras.DefaultCopyOptions)
+	if err != nil {
+		return sha256.SHA256{}, fmt.Errorf("unable to push artefact upstream from staging: %w", err)
+	}
+	fmt.Printf("hash:%s\ndesc: %s\n%v", hex.EncodeToString(hash[:]), desc.Digest.String(), desc)
+	return hash, nil
 }
 
 func (s *ContainerService) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCloser, error) {
-	_, stream, err := oras.Fetch(ctx, s.repository, remoteModulePath(s.host, digest), oras.DefaultFetchOptions)
+	ref := fmt.Sprintf("ftl/modules/%s", digest)
+	registry, err := s.connectionBuilder(ref)
+	if err != nil {
+		return nil, fmt.Errorf("unable to connect to registry '%s/%s': %w", s.host, ref, err)
+	}
+	_, stream, err := oras.Fetch(ctx, registry, remoteModulePath(s.host, digest), oras.DefaultFetchOptions)
 	if err != nil {
 		return nil, fmt.Errorf("unable to download artefact: %w", err)
 	}
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index 4754188759..5419808d75 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -40,8 +40,8 @@ func (d *releasePublishCmd) Run() error {
 	conn.SetMaxOpenConns(d.MaxOpenDBConnections)
 
 	svc := artefacts.NewContainerService(artefacts.ContainerConfig{
-		Registry:   "localhost:5000",
-		Repository: "demo-repo",
+		Registry:       "localhost:5000",
+		AllowPlainHTTP: true,
 	}, conn)
 	content := uuid.New()
 	contentBytes := content[:]

From aa8f92c2b0a162c19c68783e9b59ea351c4aa0cb Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Tue, 8 Oct 2024 14:16:37 -0700
Subject: [PATCH 06/14] artefact discovery exposed to the cli

---
 backend/controller/artefacts/dal_registry.go |   6 +-
 backend/controller/artefacts/oci_registry.go | 120 +++++++++++++++----
 frontend/cli/cmd_release.go                  |  50 ++++++--
 3 files changed, 145 insertions(+), 31 deletions(-)

diff --git a/backend/controller/artefacts/dal_registry.go b/backend/controller/artefacts/dal_registry.go
index 6a1442a66d..43236a4ff0 100644
--- a/backend/controller/artefacts/dal_registry.go
+++ b/backend/controller/artefacts/dal_registry.go
@@ -94,7 +94,11 @@ func getDatabaseReleaseArtefacts(ctx context.Context, db sql.Querier, releaseID
 }
 
 func (s *Service) AddReleaseArtefact(ctx context.Context, key model.DeploymentKey, ra ReleaseArtefact) error {
-	err := s.db.AssociateArtefactWithDeployment(ctx, sql.AssociateArtefactWithDeploymentParams{
+	return addReleaseArtefacts(ctx, s.db, key, ra)
+}
+
+func addReleaseArtefacts(ctx context.Context, db sql.Querier, key model.DeploymentKey, ra ReleaseArtefact) error {
+	err := db.AssociateArtefactWithDeployment(ctx, sql.AssociateArtefactWithDeploymentParams{
 		Key:        key,
 		Digest:     ra.Artefact.Digest[:],
 		Executable: ra.Executable,
diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index 9c05a38da3..a07f741a2f 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -17,6 +17,11 @@ import (
 	"oras.land/oras-go/v2/registry/remote"
 	"oras.land/oras-go/v2/registry/remote/auth"
 	"oras.land/oras-go/v2/registry/remote/retry"
+	"strings"
+)
+
+const (
+	ModuleArtifactPrefix = "ftl/modules/"
 )
 
 type ContainerConfig struct {
@@ -27,18 +32,26 @@ type ContainerConfig struct {
 }
 
 type ContainerService struct {
-	host              string
-	connectionBuilder func(container string) (*remote.Repository, error)
+	host                  string
+	repoConnectionBuilder func(container string) (*remote.Repository, error)
 
 	// in the interim releases and artefacts will continue to be linked via the `deployment_artefacts` table
 	Handle *libdal.Handle[ContainerService]
 	db     sql.Querier
 }
 
+type ArtefactRepository struct {
+	ModuleDigest     sha256.SHA256
+	MediaType        string
+	ArtefactType     string
+	RepositoryDigest digest.Digest
+	Size             int64
+}
+
 func NewContainerService(c ContainerConfig, conn libdal.Connection) *ContainerService {
 	// Connect the registry targeting the specified container
-	connectionBuilder := func(container string) (*remote.Repository, error) {
-		ref := fmt.Sprintf("%s/%s", c.Registry, container)
+	repoConnectionBuilder := func(path string) (*remote.Repository, error) {
+		ref := fmt.Sprintf("%s/%s", c.Registry, path)
 		reg, err := remote.NewRepository(ref)
 		if err != nil {
 			return nil, fmt.Errorf("unable to connect to container registry '%s': %w", ref, err)
@@ -58,14 +71,14 @@ func NewContainerService(c ContainerConfig, conn libdal.Connection) *ContainerSe
 	}
 
 	return &ContainerService{
-		host:              c.Registry,
-		connectionBuilder: connectionBuilder,
+		host:                  c.Registry,
+		repoConnectionBuilder: repoConnectionBuilder,
 		Handle: libdal.New(conn, func(h *libdal.Handle[ContainerService]) *ContainerService {
 			return &ContainerService{
-				host:              c.Registry,
-				connectionBuilder: connectionBuilder,
-				Handle:            h,
-				db:                sql.New(h.Connection),
+				host:                  c.Registry,
+				repoConnectionBuilder: repoConnectionBuilder,
+				Handle:                h,
+				db:                    sql.New(h.Connection),
 			}
 		}),
 	}
@@ -100,11 +113,11 @@ func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha25
 	if err = ms.Tag(ctx, manifestDescriptor, tag); err != nil {
 		return sha256.SHA256{}, fmt.Errorf("unable to tag artifact: %w", err)
 	}
-	registry, err := s.connectionBuilder(ref)
+	repo, err := s.repoConnectionBuilder(ref)
 	if err != nil {
-		return sha256.SHA256{}, fmt.Errorf("unable to connect to registry '%s/%s': %w", s.host, ref, err)
+		return sha256.SHA256{}, fmt.Errorf("unable to connect to repository '%s/%s': %w", s.host, ref, err)
 	}
-	desc, err := oras.Copy(ctx, ms, tag, registry, tag, oras.DefaultCopyOptions)
+	desc, err := oras.Copy(ctx, ms, tag, repo, tag, oras.DefaultCopyOptions)
 	if err != nil {
 		return sha256.SHA256{}, fmt.Errorf("unable to push artefact upstream from staging: %w", err)
 	}
@@ -113,26 +126,93 @@ func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha25
 }
 
 func (s *ContainerService) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCloser, error) {
-	ref := fmt.Sprintf("ftl/modules/%s", digest)
-	registry, err := s.connectionBuilder(ref)
+	ref := createModuleRepositoryPathFromDigest(digest)
+	registry, err := s.repoConnectionBuilder(ref)
 	if err != nil {
 		return nil, fmt.Errorf("unable to connect to registry '%s/%s': %w", s.host, ref, err)
 	}
-	_, stream, err := oras.Fetch(ctx, registry, remoteModulePath(s.host, digest), oras.DefaultFetchOptions)
+	_, stream, err := oras.Fetch(ctx, registry, createModuleRepositoryReferenceFromDigest(s.host, digest), oras.DefaultFetchOptions)
 	if err != nil {
 		return nil, fmt.Errorf("unable to download artefact: %w", err)
 	}
 	return stream, nil
 }
 
+func (s *ContainerService) DiscoverModuleArtefacts(ctx context.Context) ([]ArtefactRepository, error) {
+	return s.DiscoverArtefacts(ctx, ModuleArtifactPrefix)
+}
+
+func (s *ContainerService) DiscoverArtefacts(ctx context.Context, prefix string) ([]ArtefactRepository, error) {
+	registry, err := remote.NewRegistry(s.host)
+	if err != nil {
+		return nil, fmt.Errorf("unable to connect to registry '%s': %w", s.host, err)
+	}
+	registry.PlainHTTP = true
+	result := make([]ArtefactRepository, 0)
+	err = registry.Repositories(ctx, "", func(repos []string) error {
+		for _, path := range repos {
+			if !strings.HasPrefix(path, prefix) {
+				continue
+			}
+
+			d, err := getDigestFromModuleRepositoryPath(path)
+			if err != nil {
+				return fmt.Errorf("unable to get digest from repository path '%s': %w", path, err)
+			}
+
+			repo, err := registry.Repository(ctx, path)
+			if err != nil {
+				return fmt.Errorf("unable to connect to repository '%s': %w", path, err)
+			}
+
+			desc, err := repo.Resolve(ctx, "latest")
+			if err != nil {
+				return fmt.Errorf("unable to resolve module metadata '%s': %w", path, err)
+			}
+
+			result = append(result, ArtefactRepository{
+				ModuleDigest:     d,
+				MediaType:        desc.MediaType,
+				ArtefactType:     desc.ArtifactType,
+				RepositoryDigest: desc.Digest,
+				Size:             desc.Size,
+			})
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to discover artefacts: %w", err)
+	}
+	return result, nil
+}
+
 func (s *ContainerService) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) {
-	return nil, nil
+	return getDatabaseReleaseArtefacts(ctx, s.db, releaseID)
 }
 
 func (s *ContainerService) AddReleaseArtefact(ctx context.Context, key model.DeploymentKey, ra ReleaseArtefact) error {
-	return nil
+	return addReleaseArtefacts(ctx, s.db, key, ra)
 }
 
-func remoteModulePath(host string, digest sha256.SHA256) string {
-	return fmt.Sprintf("%s/modules/%s:latest", host, hex.EncodeToString(digest[:]))
+// createModuleRepositoryPathFromDigest creates the path to the repository, relative to the registries root
+func createModuleRepositoryPathFromDigest(digest sha256.SHA256) string {
+	return fmt.Sprintf("%s/%s:latest", ModuleArtifactPrefix, hex.EncodeToString(digest[:]))
+}
+
+// createModuleRepositoryReferenceFromDigest creates the URL used to connect to the repository
+func createModuleRepositoryReferenceFromDigest(host string, digest sha256.SHA256) string {
+	return fmt.Sprintf("%s/%s", host, createModuleRepositoryPathFromDigest(digest))
+}
+
+// getDigestFromModuleRepositoryPath extracts the digest from the module repository path; e.g. /ftl/modules/<digest>:latest
+func getDigestFromModuleRepositoryPath(repository string) (sha256.SHA256, error) {
+	slash := strings.LastIndex(repository, "/")
+	if slash == -1 {
+		return sha256.SHA256{}, fmt.Errorf("unable to parse repository '%s'", repository)
+	}
+	d, err := sha256.ParseSHA256(repository[slash+1:])
+	if err != nil {
+		return sha256.SHA256{}, fmt.Errorf("unable to parse repository digest '%s': %w", repository, err)
+	}
+	return d, nil
 }
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index 5419808d75..c5c338889d 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -32,17 +32,10 @@ type releasePublishCmd struct {
 }
 
 func (d *releasePublishCmd) Run() error {
-	conn, err := internalobservability.OpenDBAndInstrument(d.DSN)
+	svc, err := createContainerService(d.DSN, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
 	if err != nil {
-		return fmt.Errorf("failed to open DB connection: %w", err)
+		return fmt.Errorf("failed to create container service: %w", err)
 	}
-	conn.SetMaxIdleConns(d.MaxIdleDBConnections)
-	conn.SetMaxOpenConns(d.MaxOpenDBConnections)
-
-	svc := artefacts.NewContainerService(artefacts.ContainerConfig{
-		Registry:       "localhost:5000",
-		AllowPlainHTTP: true,
-	}, conn)
 	content := uuid.New()
 	contentBytes := content[:]
 	_, err = svc.Upload(context.Background(), artefacts.Artefact{
@@ -59,8 +52,45 @@ func (d *releasePublishCmd) Run() error {
 }
 
 type releaseListCmd struct {
+	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
+	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
+	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
 }
 
 func (d *releaseListCmd) Run() error {
-	return fmt.Errorf("release list not implemented")
+	svc, err := createContainerService(d.DSN, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
+	if err != nil {
+		return fmt.Errorf("failed to create container service: %w", err)
+	}
+	modules, err := svc.DiscoverModuleArtefacts(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to discover module artefacts: %w", err)
+	}
+	if len(modules) == 0 {
+		fmt.Println("No module artefacts found.")
+		return nil
+	}
+
+	format := "    Digest        : %s\n    Size          : %-7d\n    Repo Digest   : %s\n    Media Type    : %s\n    Artefact Type : %s\n"
+	fmt.Printf("Found %d module artefacts:\n", len(modules))
+	for i, m := range modules {
+		fmt.Printf("\n\033[31m  Artefact %d\033[0m\n\n", i)
+		fmt.Printf(format, m.ModuleDigest, m.Size, m.RepositoryDigest, m.MediaType, m.ArtefactType)
+	}
+
+	return nil
+}
+
+func createContainerService(dsn string, maxOpenConn int, maxIdleCon int) (*artefacts.ContainerService, error) {
+	conn, err := internalobservability.OpenDBAndInstrument(dsn)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open DB connection: %w", err)
+	}
+	conn.SetMaxIdleConns(maxIdleCon)
+	conn.SetMaxOpenConns(maxOpenConn)
+
+	return artefacts.NewContainerService(artefacts.ContainerConfig{
+		Registry:       "localhost:5000",
+		AllowPlainHTTP: true,
+	}, conn), nil
 }

From 941e5a310516fa87ebdcbfd30376e1df32c5fb97 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Tue, 8 Oct 2024 14:52:12 -0700
Subject: [PATCH 07/14] completed oci Registry implementation

---
 backend/controller/artefacts/oci_registry.go | 21 +++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index a07f741a2f..c0ac5ff3bb 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -85,7 +85,26 @@ func NewContainerService(c ContainerConfig, conn libdal.Connection) *ContainerSe
 }
 
 func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.SHA256) (keys []ArtefactKey, missing []sha256.SHA256, err error) {
-	return nil, nil, nil
+	set := make(map[sha256.SHA256]bool)
+	for _, d := range digests {
+		set[d] = true
+	}
+	modules, err := s.DiscoverModuleArtefacts(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to discover module artefacts: %w", err)
+	}
+	keys = make([]ArtefactKey, 0)
+	for _, m := range modules {
+		if set[m.ModuleDigest] {
+			keys = append(keys, ArtefactKey{Digest: m.ModuleDigest})
+			delete(set, m.ModuleDigest)
+		}
+	}
+	missing = make([]sha256.SHA256, 0)
+	for d, _ := range set {
+		missing = append(missing, d)
+	}
+	return keys, missing, nil
 }
 
 func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha256.SHA256, error) {

From bc52571847929a27b09fa00f046210915b289483 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Tue, 8 Oct 2024 15:41:22 -0700
Subject: [PATCH 08/14] fix cli spec

---
 frontend/cli/cmd_release.go | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index c5c338889d..7dd02e577d 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -12,17 +12,8 @@ import (
 )
 
 type releaseCmd struct {
-	Describe releaseDescribeCmd `cmd:"" help:"Describes the specified release."`
-	Publish  releasePublishCmd  `cmd:"" help:"Packages the project into a release and publishes it."`
-	List     releaseListCmd     `cmd:"" help:"Lists all published releases."`
-}
-
-type releaseDescribeCmd struct {
-	Digest string `arg:"" help:"Digest of the target release."`
-}
-
-func (d *releaseDescribeCmd) Run() error {
-	return fmt.Errorf("release describe not implemented")
+	Publish releasePublishCmd `cmd:"" help:"Packages the project into a release and publishes it."`
+	List    releaseListCmd    `cmd:"" help:"Lists all published releases."`
 }
 
 type releasePublishCmd struct {
@@ -74,7 +65,7 @@ func (d *releaseListCmd) Run() error {
 	format := "    Digest        : %s\n    Size          : %-7d\n    Repo Digest   : %s\n    Media Type    : %s\n    Artefact Type : %s\n"
 	fmt.Printf("Found %d module artefacts:\n", len(modules))
 	for i, m := range modules {
-		fmt.Printf("\n\033[31m  Artefact %d\033[0m\n\n", i)
+		fmt.Printf("\033[31m  Artefact %d\033[0m\n", i)
 		fmt.Printf(format, m.ModuleDigest, m.Size, m.RepositoryDigest, m.MediaType, m.ArtefactType)
 	}
 

From 9521ee0ea3f0f9d2224ab1adeec554854a403f44 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Tue, 8 Oct 2024 16:15:49 -0700
Subject: [PATCH 09/14] shift external port for registry to avoid conflict

---
 backend/controller/artefacts/oci_registry.go |  2 +-
 docker-compose.yml                           |  2 +-
 frontend/cli/cmd_release.go                  | 10 ++++++----
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index c0ac5ff3bb..6715c013df 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -101,7 +101,7 @@ func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.
 		}
 	}
 	missing = make([]sha256.SHA256, 0)
-	for d, _ := range set {
+	for d := range set {
 		missing = append(missing, d)
 	}
 	return keys, missing, nil
diff --git a/docker-compose.yml b/docker-compose.yml
index e509d23e71..58fc482c75 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
   registry:
     image: registry:2
     ports:
-      -  "5000:5000"
+      -  "5001:5000"
     volumes:
       - ./.registry:/var/lib/registry
 
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index 7dd02e577d..fd4295ba76 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -17,13 +17,14 @@ type releaseCmd struct {
 }
 
 type releasePublishCmd struct {
+	Registry             string `help:"Registry host:port" default:"127.0.0.1:5001"`
 	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
 	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
 	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
 }
 
 func (d *releasePublishCmd) Run() error {
-	svc, err := createContainerService(d.DSN, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
+	svc, err := createContainerService(d.DSN, d.Registry, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
 	if err != nil {
 		return fmt.Errorf("failed to create container service: %w", err)
 	}
@@ -43,13 +44,14 @@ func (d *releasePublishCmd) Run() error {
 }
 
 type releaseListCmd struct {
+	Registry             string `help:"Registry host:port" default:"127.0.0.1:5001"`
 	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
 	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
 	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
 }
 
 func (d *releaseListCmd) Run() error {
-	svc, err := createContainerService(d.DSN, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
+	svc, err := createContainerService(d.DSN, d.Registry, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
 	if err != nil {
 		return fmt.Errorf("failed to create container service: %w", err)
 	}
@@ -72,7 +74,7 @@ func (d *releaseListCmd) Run() error {
 	return nil
 }
 
-func createContainerService(dsn string, maxOpenConn int, maxIdleCon int) (*artefacts.ContainerService, error) {
+func createContainerService(dsn string, reg string, maxOpenConn int, maxIdleCon int) (*artefacts.ContainerService, error) {
 	conn, err := internalobservability.OpenDBAndInstrument(dsn)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open DB connection: %w", err)
@@ -81,7 +83,7 @@ func createContainerService(dsn string, maxOpenConn int, maxIdleCon int) (*artef
 	conn.SetMaxOpenConns(maxOpenConn)
 
 	return artefacts.NewContainerService(artefacts.ContainerConfig{
-		Registry:       "localhost:5000",
+		Registry:       reg,
 		AllowPlainHTTP: true,
 	}, conn), nil
 }

From 2d0618a28e75d4338b8a60c6ae266b6abdf6bcc6 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Wed, 9 Oct 2024 18:49:49 +0000
Subject: [PATCH 10/14] chore(autofmt): Automated formatting

---
 backend/controller/artefacts/oci_registry.go | 14 ++++++++------
 go.mod                                       |  4 ++--
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index 6715c013df..66b3d1112b 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -5,19 +5,21 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
-	"github.com/TBD54566975/ftl/backend/libdal"
-	"github.com/TBD54566975/ftl/internal/model"
-	"github.com/TBD54566975/ftl/internal/sha256"
+	"io"
+	"strings"
+
 	"github.com/opencontainers/go-digest"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
-	"io"
 	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/content/memory"
 	"oras.land/oras-go/v2/registry/remote"
 	"oras.land/oras-go/v2/registry/remote/auth"
 	"oras.land/oras-go/v2/registry/remote/retry"
-	"strings"
+
+	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
+	"github.com/TBD54566975/ftl/backend/libdal"
+	"github.com/TBD54566975/ftl/internal/model"
+	"github.com/TBD54566975/ftl/internal/sha256"
 )
 
 const (
diff --git a/go.mod b/go.mod
index 8cb9e100cf..15d2610b31 100644
--- a/go.mod
+++ b/go.mod
@@ -43,6 +43,8 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/mattn/go-isatty v0.0.20
 	github.com/multiformats/go-base36 v0.2.0
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.1.0
 	github.com/otiai10/copy v1.14.0
 	github.com/posener/complete v1.2.3
 	github.com/radovskyb/watcher v1.0.7
@@ -131,8 +133,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/onsi/gomega v1.33.1 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pkoukk/tiktoken-go v0.1.6 // indirect

From 6b87750a3b616af85687bf5c465280d11f90f653 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Wed, 9 Oct 2024 12:49:36 -0700
Subject: [PATCH 11/14] collect correct media and artifact type metadata

---
 backend/controller/artefacts/oci_registry.go | 21 +++++++++++---------
 frontend/cli/cmd_release.go                  |  3 ++-
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index 66b3d1112b..afb5607038 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"strings"
@@ -138,11 +139,9 @@ func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha25
 	if err != nil {
 		return sha256.SHA256{}, fmt.Errorf("unable to connect to repository '%s/%s': %w", s.host, ref, err)
 	}
-	desc, err := oras.Copy(ctx, ms, tag, repo, tag, oras.DefaultCopyOptions)
-	if err != nil {
+	if _, err = oras.Copy(ctx, ms, tag, repo, tag, oras.DefaultCopyOptions); err != nil {
 		return sha256.SHA256{}, fmt.Errorf("unable to push artefact upstream from staging: %w", err)
 	}
-	fmt.Printf("hash:%s\ndesc: %s\n%v", hex.EncodeToString(hash[:]), desc.Digest.String(), desc)
 	return hash, nil
 }
 
@@ -175,26 +174,30 @@ func (s *ContainerService) DiscoverArtefacts(ctx context.Context, prefix string)
 			if !strings.HasPrefix(path, prefix) {
 				continue
 			}
-
 			d, err := getDigestFromModuleRepositoryPath(path)
 			if err != nil {
 				return fmt.Errorf("unable to get digest from repository path '%s': %w", path, err)
 			}
-
 			repo, err := registry.Repository(ctx, path)
 			if err != nil {
 				return fmt.Errorf("unable to connect to repository '%s': %w", path, err)
 			}
-
 			desc, err := repo.Resolve(ctx, "latest")
 			if err != nil {
 				return fmt.Errorf("unable to resolve module metadata '%s': %w", path, err)
 			}
-
+			_, data, err := oras.FetchBytes(ctx, repo, desc.Digest.String(), oras.DefaultFetchBytesOptions)
+			if err != nil {
+				return fmt.Errorf("unable to fetch module metadata '%s': %w", path, err)
+			}
+			var manifest v1.Manifest
+			if err := json.Unmarshal(data, &manifest); err != nil {
+				return fmt.Errorf("unable to unmarshal module metadata '%s': %w", path, err)
+			}
 			result = append(result, ArtefactRepository{
 				ModuleDigest:     d,
-				MediaType:        desc.MediaType,
-				ArtefactType:     desc.ArtifactType,
+				MediaType:        manifest.Layers[0].MediaType,
+				ArtefactType:     manifest.ArtifactType,
 				RepositoryDigest: desc.Digest,
 				Size:             desc.Size,
 			})
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index fd4295ba76..4037a3c8d8 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -30,7 +30,7 @@ func (d *releasePublishCmd) Run() error {
 	}
 	content := uuid.New()
 	contentBytes := content[:]
-	_, err = svc.Upload(context.Background(), artefacts.Artefact{
+	hash, err := svc.Upload(context.Background(), artefacts.Artefact{
 		Digest: sha256.Sum256(contentBytes),
 		Metadata: artefacts.Metadata{
 			Path: fmt.Sprintf("random/%s", content),
@@ -40,6 +40,7 @@ func (d *releasePublishCmd) Run() error {
 	if err != nil {
 		return fmt.Errorf("failed upload artefact: %w", err)
 	}
+	fmt.Printf("Artefact published with hash: \033[31m%s\033[0m\n", hash)
 	return nil
 }
 

From 1d4817224c872dd86e07d8fe4c8f9bcb2dea41b1 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Thu, 10 Oct 2024 15:43:42 -0700
Subject: [PATCH 12/14] applying pr feedback

---
 backend/controller/artefacts/oci_registry.go |  8 ++---
 frontend/cli/cmd_release.go                  | 32 +++++++++-----------
 2 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index afb5607038..8e7e1950b1 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -28,10 +28,10 @@ const (
 )
 
 type ContainerConfig struct {
-	Registry       string
-	Username       string
-	Password       string
-	AllowPlainHTTP bool
+	Registry       string `help:"OCI container registry host:port" env:"FTL_ARTEFACTS_REGISTRY"`
+	Username       string `help:"OCI container registry username" env:"FTL_ARTEFACTS_USER"`
+	Password       string `help:"OCI container registry password" env:"FTL_ARTEFACTS_PWD"`
+	AllowPlainHTTP bool   `help:"Allows OCI container requests to accept plain HTTP responses" env:"FTL_ARTEFACTS_ALLOW_HTTP"`
 }
 
 type ContainerService struct {
diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index 4037a3c8d8..1c5ff5da3a 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
-
 	"github.com/google/uuid"
 
 	"github.com/TBD54566975/ftl/backend/controller/artefacts"
@@ -12,19 +11,20 @@ import (
 )
 
 type releaseCmd struct {
+	Registry             string `help:"Registry host:port" default:"127.0.0.1:5001"`
+	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
+	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
+	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
+
 	Publish releasePublishCmd `cmd:"" help:"Packages the project into a release and publishes it."`
 	List    releaseListCmd    `cmd:"" help:"Lists all published releases."`
 }
 
 type releasePublishCmd struct {
-	Registry             string `help:"Registry host:port" default:"127.0.0.1:5001"`
-	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
-	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
-	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
 }
 
-func (d *releasePublishCmd) Run() error {
-	svc, err := createContainerService(d.DSN, d.Registry, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
+func (d *releasePublishCmd) Run(release *releaseCmd) error {
+	svc, err := createContainerService(release)
 	if err != nil {
 		return fmt.Errorf("failed to create container service: %w", err)
 	}
@@ -45,14 +45,10 @@ func (d *releasePublishCmd) Run() error {
 }
 
 type releaseListCmd struct {
-	Registry             string `help:"Registry host:port" default:"127.0.0.1:5001"`
-	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
-	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
-	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
 }
 
-func (d *releaseListCmd) Run() error {
-	svc, err := createContainerService(d.DSN, d.Registry, d.MaxOpenDBConnections, d.MaxIdleDBConnections)
+func (d *releaseListCmd) Run(release *releaseCmd) error {
+	svc, err := createContainerService(release)
 	if err != nil {
 		return fmt.Errorf("failed to create container service: %w", err)
 	}
@@ -75,16 +71,16 @@ func (d *releaseListCmd) Run() error {
 	return nil
 }
 
-func createContainerService(dsn string, reg string, maxOpenConn int, maxIdleCon int) (*artefacts.ContainerService, error) {
-	conn, err := internalobservability.OpenDBAndInstrument(dsn)
+func createContainerService(release *releaseCmd) (*artefacts.ContainerService, error) {
+	conn, err := internalobservability.OpenDBAndInstrument(release.DSN)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open DB connection: %w", err)
 	}
-	conn.SetMaxIdleConns(maxIdleCon)
-	conn.SetMaxOpenConns(maxOpenConn)
+	conn.SetMaxIdleConns(release.MaxIdleDBConnections)
+	conn.SetMaxOpenConns(release.MaxOpenDBConnections)
 
 	return artefacts.NewContainerService(artefacts.ContainerConfig{
-		Registry:       reg,
+		Registry:       release.Registry,
 		AllowPlainHTTP: true,
 	}, conn), nil
 }

From f026f307c2860ed168f852d65146af1c7a8901a7 Mon Sep 17 00:00:00 2001
From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com>
Date: Thu, 10 Oct 2024 16:05:14 -0700
Subject: [PATCH 13/14] applying pr feedback

---
 backend/controller/artefacts/oci_registry.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/backend/controller/artefacts/oci_registry.go b/backend/controller/artefacts/oci_registry.go
index 8e7e1950b1..b8cdb881d3 100644
--- a/backend/controller/artefacts/oci_registry.go
+++ b/backend/controller/artefacts/oci_registry.go
@@ -111,12 +111,11 @@ func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.
 }
 
 func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha256.SHA256, error) {
-	hash := sha256.Sum(artefact.Content)
-	ref := fmt.Sprintf("ftl/modules/%s", hash)
+	ref := fmt.Sprintf("ftl/modules/%s", artefact.Digest)
 	ms := memory.New()
 	mediaDescriptor := v1.Descriptor{
 		MediaType: "application/ftl.module.v1",
-		Digest:    digest.NewDigestFromBytes(digest.SHA256, hash[:]),
+		Digest:    digest.NewDigestFromBytes(digest.SHA256, artefact.Digest[:]),
 		Size:      int64(len(artefact.Content)),
 	}
 	err := ms.Push(ctx, mediaDescriptor, bytes.NewReader(artefact.Content))
@@ -142,7 +141,7 @@ func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha25
 	if _, err = oras.Copy(ctx, ms, tag, repo, tag, oras.DefaultCopyOptions); err != nil {
 		return sha256.SHA256{}, fmt.Errorf("unable to push artefact upstream from staging: %w", err)
 	}
-	return hash, nil
+	return artefact.Digest, nil
 }
 
 func (s *ContainerService) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCloser, error) {

From 3d793c442027121284c06ae6e8c245d63982feaa Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Thu, 10 Oct 2024 23:07:19 +0000
Subject: [PATCH 14/14] chore(autofmt): Automated formatting

---
 frontend/cli/cmd_release.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frontend/cli/cmd_release.go b/frontend/cli/cmd_release.go
index 1c5ff5da3a..0c5f23f525 100644
--- a/frontend/cli/cmd_release.go
+++ b/frontend/cli/cmd_release.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
+
 	"github.com/google/uuid"
 
 	"github.com/TBD54566975/ftl/backend/controller/artefacts"