Skip to content

Commit

Permalink
Record latest version, title and description if the connector metadat…
Browse files Browse the repository at this point in the history 
…a changes (#383)

Every time the connector's metadata.json file changes, the
`latest_version`, `title` and `description` will be updated to the
latest values in the registry DB.
  • Loading branch information
@codingkarthik
codingkarthik authored Nov 28, 2024
1 parent dc9b3a6 commit caaf6cc
Show file tree
Hide file tree
Showing 5 changed files with 129 additions and 38 deletions.
111 changes: 87 additions & 24 deletions registry-automation/cmd/ci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,9 @@ func buildContext() Context {
}

type fileProcessor struct {
regex *regexp.Regexp
process func(matches []string, file string)
regex *regexp.Regexp
newFileHandler func(matches []string, file string)
modifiedFileHandler func(matches []string, file string) error
}

// processChangedFiles categorizes changes in connector files within a registry system.
Expand Down Expand Up @@ -154,59 +155,78 @@ func processChangedFiles(changedFiles ChangedFiles) ProcessedChangedFiles {
NewConnectors: make(map[Connector]MetadataFile),
NewLogos: make(map[Connector]string),
NewReadmes: make(map[Connector]string),
ModifiedConnectors: make(map[Connector]MetadataFile),
}

processors := []fileProcessor{
{
regex: regexp.MustCompile(`^registry/([^/]+)/([^/]+)/metadata.json$`),
process: func(matches []string, file string) {
// IsNew is set to true because we are processing newly added metadata.json
newFileHandler: func(matches []string, file string) {
connector := Connector{Name: matches[2], Namespace: matches[1]}
result.NewConnectors[connector] = MetadataFile(file)
fmt.Printf("Processing metadata file for connector: %s\n", connector.Name)
fmt.Printf("Processing metadata file for new connector: %s\n", connector.Name)
},
modifiedFileHandler: func(matches []string, file string) error {
connector := Connector{Name: matches[2], Namespace: matches[1]}
result.ModifiedConnectors[connector] = MetadataFile(file)
fmt.Printf("Processing metadata file for modified connector: %s\n", connector.Name)
return nil
},
},
{
regex: regexp.MustCompile(`^registry/([^/]+)/([^/]+)/logo\.(png|svg)$`),
process: func(matches []string, file string) {
newFileHandler: func(matches []string, file string) {
connector := Connector{Name: matches[2], Namespace: matches[1]}
result.NewLogos[connector] = file
fmt.Printf("Processing logo file for connector: %s\n", connector.Name)
fmt.Printf("Processing logo file for new connector: %s\n", connector.Name)
},
modifiedFileHandler: func(matches []string, file string) error {
connector := Connector{Name: matches[2], Namespace: matches[1]}
result.ModifiedLogos[connector] = file
fmt.Printf("Processing logo file for modified connector: %s\n", connector.Name)
return nil
},
},
{
regex: regexp.MustCompile(`^registry/([^/]+)/([^/]+)/README\.md$`),
process: func(matches []string, file string) {
newFileHandler: func(matches []string, file string) {
connector := Connector{Name: matches[2], Namespace: matches[1]}
result.NewReadmes[connector] = file
fmt.Printf("Processing README file for connector: %s\n", connector.Name)
fmt.Printf("Processing README file for new connector: %s\n", connector.Name)
},
modifiedFileHandler: func(matches []string, file string) error {
connector := Connector{Name: matches[2], Namespace: matches[1]}
result.ModifiedReadmes[connector] = file
fmt.Printf("Processing README file for modified connector: %s\n", connector.Name)
return nil
},
},
{
regex: regexp.MustCompile(`^registry/([^/]+)/([^/]+)/releases/([^/]+)/connector-packaging\.json$`),
process: func(matches []string, file string) {
newFileHandler: func(matches []string, file string) {
connector := Connector{Name: matches[2], Namespace: matches[1]}
version := matches[3]
if _, exists := result.NewConnectorVersions[connector]; !exists {
result.NewConnectorVersions[connector] = make(map[string]string)
}
result.NewConnectorVersions[connector][version] = file
},
modifiedFileHandler: func(matches []string, file string) error {
return fmt.Errorf("Connector packaging files (%s) are immutable and should not be changed", file)
},
},
}

processFile := func(file string, isModified bool) {
for _, processor := range processors {
if matches := processor.regex.FindStringSubmatch(file); matches != nil {
if isModified {
connector := Connector{Name: matches[2], Namespace: matches[1]}
if processor.regex.String() == processors[1].regex.String() {
result.ModifiedLogos[connector] = file
} else if processor.regex.String() == processors[2].regex.String() {
result.ModifiedReadmes[connector] = file
processError := processor.modifiedFileHandler(matches, file)
if processError != nil {
fmt.Printf("Error processing modified %s file: %s: %v\n", matches[2], file, processError)
}
} else {
processor.process(matches, file)
processor.newFileHandler(matches, file)
}
return
}
Expand All @@ -225,6 +245,35 @@ func processChangedFiles(changedFiles ChangedFiles) ProcessedChangedFiles {
return result
}

// processModifiedConnectors processes the modified connectors and updates the connector metadata in the registry
// This function updates the registry with the latest version, title, and description of the connector
func processModifiedConnector(metadataFile MetadataFile, connector Connector) (ConnectorOverviewUpdate, error) {
// Iterate over the modified connectors and update the connectors in the registry
var connectorOverviewUpdate ConnectorOverviewUpdate
connectorMetadata, err := readJSONFile[ConnectorMetadata](string(metadataFile))
if err != nil {
return connectorOverviewUpdate, fmt.Errorf("Failed to parse the connector metadata file: %v", err)
}
connectorOverviewUpdate = ConnectorOverviewUpdate{
Set: struct {
Docs *string `json:"docs,omitempty"`
Logo *string `json:"logo,omitempty"`
LatestVersion *string `json:"latest_version,omitempty"`
Title *string `json:"title,omitempty"`
Description *string `json:"description,omitempty"`
}{
LatestVersion: &connectorMetadata.Overview.LatestVersion,
Title: &connectorMetadata.Overview.Title,
Description: &connectorMetadata.Overview.Description,
},
Where: WhereClause{
ConnectorName: connector.Name,
ConnectorNamespace: connector.Namespace,
},
}
return connectorOverviewUpdate, nil
}

func processNewConnector(ciCtx Context, connector Connector, metadataFile MetadataFile) (ConnectorOverviewInsert, HubRegistryConnectorInsertInput, error) {
// Process the newly added connector
// Get the string value from metadataFile
Expand Down Expand Up @@ -277,14 +326,15 @@ func processNewConnector(ciCtx Context, connector Connector, metadataFile Metada
}

connectorOverviewAndAuthor = ConnectorOverviewInsert{
Name: connector.Name,
Namespace: connector.Namespace,
Docs: string(docs),
Logo: uploadedLogoUrl,
Title: connectorMetadata.Overview.Title,
Description: connectorMetadata.Overview.Description,
IsVerified: connectorMetadata.IsVerified,
IsHosted: connectorMetadata.IsHostedByHasura,
Name: connector.Name,
Namespace: connector.Namespace,
Docs: string(docs),
Logo: uploadedLogoUrl,
Title: connectorMetadata.Overview.Title,
Description: connectorMetadata.Overview.Description,
IsVerified: connectorMetadata.IsVerified,
IsHosted: connectorMetadata.IsHostedByHasura,
LatestVersion: connectorMetadata.Overview.LatestVersion,
Author: ConnectorAuthorNestedInsert{
Data: ConnectorAuthor{
Name: connectorMetadata.Author.Name,
Expand Down Expand Up @@ -329,6 +379,7 @@ func runCI(cmd *cobra.Command, args []string) {
modifiedReadmes := processChangedFiles.ModifiedReadmes

newlyAddedConnectors := processChangedFiles.NewConnectors
modifiedConnectors := processChangedFiles.ModifiedConnectors

var newConnectorsToBeAdded NewConnectorsInsertInput
newConnectorsToBeAdded.HubRegistryConnectors = make([]HubRegistryConnectorInsertInput, 0)
Expand Down Expand Up @@ -357,6 +408,18 @@ func runCI(cmd *cobra.Command, args []string) {

}

if len(modifiedConnectors) > 0 {
fmt.Println("Modified connectors: ", modifiedConnectors)
// Process the modified connectors
for connector, metadataFile := range modifiedConnectors {
connectorOverviewUpdate, err := processModifiedConnector(metadataFile, connector)
if err != nil {
log.Fatalf("Failed to process the modified connector: %s/%s, Error: %v", connector.Namespace, connector.Name, err)
}
connectorOverviewUpdates = append(connectorOverviewUpdates, connectorOverviewUpdate)
}
}

if len(newlyAddedConnectorVersions) > 0 {
newlyAddedConnectors := make(map[Connector]bool)
for connector := range newlyAddedConnectorVersions {
Expand Down
21 changes: 21 additions & 0 deletions registry-automation/cmd/ci_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ func TestProcessChangedFiles(t *testing.T) {
NewConnectors: map[Connector]MetadataFile{{Name: "connector1", Namespace: "namespace1"}: "registry/namespace1/connector1/metadata.json"},
NewLogos: map[Connector]string{},
NewReadmes: map[Connector]string{},
ModifiedConnectors: map[Connector]MetadataFile{},
},
},
{
Expand All @@ -91,6 +92,26 @@ func TestProcessChangedFiles(t *testing.T) {
NewConnectors: map[Connector]MetadataFile{},
NewLogos: map[Connector]string{},
NewReadmes: map[Connector]string{},
ModifiedConnectors: map[Connector]MetadataFile{},
},
},
{
name: "Modified logo and README and metadata",
changedFiles: ChangedFiles{
Modified: []string{
"registry/namespace1/connector1/logo.png",
"registry/namespace1/connector1/README.md",
"registry/namespace1/connector1/metadata.json",
},
},
expected: ProcessedChangedFiles{
NewConnectorVersions: map[Connector]map[string]string{},
ModifiedLogos: map[Connector]string{{Name: "connector1", Namespace: "namespace1"}: "registry/namespace1/connector1/logo.png"},
ModifiedReadmes: map[Connector]string{{Name: "connector1", Namespace: "namespace1"}: "registry/namespace1/connector1/README.md"},
NewConnectors: map[Connector]MetadataFile{},
NewLogos: map[Connector]string{},
NewReadmes: map[Connector]string{},
ModifiedConnectors: map[Connector]MetadataFile{{Name: "connector1", Namespace: "namespace1"}: "registry/namespace1/connector1/metadata.json"},
},
},
}
Expand Down
19 changes: 10 additions & 9 deletions registry-automation/cmd/registry_db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,15 +166,16 @@ type ConnectorAuthorNestedInsert struct {
}

type ConnectorOverviewInsert struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Title string `json:"title"`
Description string `json:"description"`
Logo string `json:"logo"`
Docs string `json:"docs"`
IsVerified bool `json:"is_verified"`
IsHosted bool `json:"is_hosted_by_hasura"`
Author ConnectorAuthorNestedInsert `json:"author"`
Namespace string `json:"namespace"`
Name string `json:"name"`
Title string `json:"title"`
Description string `json:"description"`
Logo string `json:"logo"`
Docs string `json:"docs"`
IsVerified bool `json:"is_verified"`
IsHosted bool `json:"is_hosted_by_hasura"`
LatestVersion string `json:"latest_version"`
Author ConnectorAuthorNestedInsert `json:"author"`
}

type ConnectorAuthor struct {
Expand Down
12 changes: 9 additions & 3 deletions registry-automation/cmd/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,11 @@ func (wc WhereClause) MarshalJSON() ([]byte, error) {

type ConnectorOverviewUpdate struct {
Set struct {
Docs *string `json:"docs,omitempty"`
Logo *string `json:"logo,omitempty"`
Docs *string `json:"docs,omitempty"`
Logo *string `json:"logo,omitempty"`
LatestVersion *string `json:"latest_version,omitempty"`
Title *string `json:"title,omitempty"`
Description *string `json:"description,omitempty"`
} `json:"_set"`
Where WhereClause `json:"where"`
}
Expand Down Expand Up @@ -90,7 +93,6 @@ type ConnectorMetadata struct {
Homepage string `json:"homepage"`
Name string `json:"name"`
} `json:"author"`

IsVerified bool `json:"is_verified"`
IsHostedByHasura bool `json:"is_hosted_by_hasura"`
HasuraHubConnector struct {
Expand Down Expand Up @@ -125,6 +127,7 @@ type ProcessedChangedFiles struct {
NewConnectors NewConnectors
NewLogos NewLogos
NewReadmes NewReadmes
ModifiedConnectors ModifiedMetadata
}

type GraphQLClientInterface interface {
Expand Down Expand Up @@ -172,6 +175,9 @@ type Connector struct {

type NewConnectorVersions map[Connector]map[string]string

// ModifiedMetadata represents the modified metadata in the PR, the key is the connector name and the value is the path to the modified metadata
type ModifiedMetadata map[Connector]MetadataFile

// ModifiedLogos represents the modified logos in the PR, the key is the connector name and the value is the path to the modified logo
type ModifiedLogos map[Connector]string

Expand Down
4 changes: 2 additions & 2 deletions registry/hasura/azure-cosmos/metadata.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
"overview": {
"namespace": "hasura",
"description": "Connect to a Azure Cosmos DB for NoSQL and expose it to Hasura v3 Project",
"title": "Azure Cosmos DB for NoSQL Connector",
"title": "Azure Cosmos DB for NoSQL",
"logo": "logo.png",
"tags": [],
"latest_version": "v0.1.6"
"latest_version": "v0.2.0"
},
"author": {
"support_email": "[email protected]",
Expand Down

0 comments on commit caaf6cc

Please sign in to comment.