Add substreams codegen subgraph cmd into branch

README.md

@@ -4,8 +4,6 @@
 
 # Substreams
 
-> Developer preview
-
 Substreams is a powerful blockchain indexing technology, developed for The Graph Network.
 
 Substreams enables developers to write Rust modules, composing data streams alongside the community, and provides extremely high performance indexing by virtue of parallelization, in a streaming-first fashion.

cmd/substreams/codegen.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/streamingfast/substreams/codegen"
+
+func init() {
+	rootCmd.AddCommand(codegen.Cmd)
+}

cmd/substreams/init.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"time"
 
-	connect "connectrpc.com/connect"
+	"connectrpc.com/connect"
 	"github.com/charmbracelet/glamour"
 	"github.com/charmbracelet/huh"
 	"github.com/charmbracelet/huh/spinner"

codegen/chainconfig.go

@@ -0,0 +1,94 @@
+package codegen
+
+type ChainConfig struct {
+	ID               string // Public
+	DisplayName      string // Public
+	ExplorerLink     string
+	ApiEndpoint      string
+	FirehoseEndpoint string
+	Network          string
+	SupportsCalls    bool
+}
+
+var ChainConfigByID = map[string]*ChainConfig{
+	"mainnet": {
+		DisplayName:      "Ethereum Mainnet",
+		ExplorerLink:     "https://etherscan.io",
+		ApiEndpoint:      "https://api.etherscan.io",
+		FirehoseEndpoint: "mainnet.eth.streamingfast.io:443",
+		Network:          "mainnet",
+		SupportsCalls:    true,
+	},
+	"bnb": {
+		DisplayName:      "BNB",
+		ExplorerLink:     "https://bscscan.com",
+		ApiEndpoint:      "https://api.bscscan.com",
+		FirehoseEndpoint: "bnb.streamingfast.io:443",
+		Network:          "bsc",
+		SupportsCalls:    true,
+	},
+	"polygon": {
+		DisplayName:      "Polygon",
+		ExplorerLink:     "https://polygonscan.com",
+		ApiEndpoint:      "https://api.polygonscan.com",
+		FirehoseEndpoint: "polygon.streamingfast.io:443",
+		Network:          "polygon",
+		SupportsCalls:    true,
+	},
+	"amoy": {
+		DisplayName:      "Polygon Amoy Testnet",
+		ExplorerLink:     "https://www.okx.com/web3/explorer/amoy",
+		ApiEndpoint:      "",
+		FirehoseEndpoint: "amoy.substreams.pinax.network:443",
+		Network:          "amoy",
+		SupportsCalls:    true,
+	},
+	"arbitrum": {
+		DisplayName:      "Arbitrum",
+		ExplorerLink:     "https://arbiscan.io",
+		ApiEndpoint:      "https://api.arbiscan.io",
+		FirehoseEndpoint: "arb-one.streamingfast.io:443",
+		Network:          "arbitrum",
+		SupportsCalls:    true,
+	},
+	"holesky": {
+		DisplayName:      "Holesky",
+		ExplorerLink:     "https://holesky.etherscan.io/",
+		ApiEndpoint:      "https://api-holesky.etherscan.io",
+		FirehoseEndpoint: "holesky.eth.streamingfast.io:443",
+		Network:          "holesky",
+		SupportsCalls:    true,
+	},
+	"sepolia": {
+		DisplayName:      "Sepolia Testnet",
+		ExplorerLink:     "https://sepolia.etherscan.io",
+		ApiEndpoint:      "https://api-sepolia.etherscan.io",
+		FirehoseEndpoint: "sepolia.streamingfast.io:443",
+		Network:          "sepolia",
+		SupportsCalls:    true,
+	},
+	"optimism": {
+		DisplayName:      "Optimism Mainnet",
+		ExplorerLink:     "https://optimistic.etherscan.io",
+		ApiEndpoint:      "https://api-optimistic.etherscan.io",
+		FirehoseEndpoint: "opt-mainnet.streamingfast.io:443",
+		Network:          "optimism",
+		SupportsCalls:    false,
+	},
+	"avalanche": {
+		DisplayName:      "Avalanche C-chain",
+		ExplorerLink:     "https://subnets.avax.network/c-chain",
+		ApiEndpoint:      "",
+		FirehoseEndpoint: "avalanche-mainnet.streamingfast.io:443",
+		Network:          "avalanche",
+		SupportsCalls:    false,
+	},
+	"chapel": {
+		DisplayName:      "BNB Chapel Testnet",
+		ExplorerLink:     "https://testnet.bscscan.com/",
+		ApiEndpoint:      "",
+		FirehoseEndpoint: "chapel.substreams.pinax.network:443",
+		Network:          "chapel",
+		SupportsCalls:    true,
+	},
+}

codegen/cmd.go

@@ -0,0 +1,14 @@
+package codegen
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var Cmd = &cobra.Command{Use: "codegen", Short: "Code generator for substreams"}
+
+func init() {
+	SubgraphCmd.Flags().Bool("with-dev-env", false, "generate graph node dev environment")
+
+	Cmd.AddCommand(SubgraphCmd)
+	Cmd.AddCommand(SQLCmd)
+}

codegen/helpers.go

@@ -0,0 +1,230 @@
+package codegen
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/streamingfast/substreams/manifest"
+
+	"github.com/charmbracelet/huh"
+
+	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
+	"google.golang.org/protobuf/types/descriptorpb"
+)
+
+const outputTypeSQL = "sql"
+const outputTypeSubgraph = "subgraph"
+
+func getModule(pkg *pbsubstreams.Package, moduleName string) (*pbsubstreams.Module, error) {
+	existingModules := pkg.GetModules().GetModules()
+	for _, module := range existingModules {
+		if (module.Name) == moduleName {
+			return module, nil
+		}
+	}
+
+	return nil, fmt.Errorf("module %q does not exists", moduleName)
+}
+
+func searchForMessageTypeIntoPackage(pkg *pbsubstreams.Package, outputType string) (*descriptorpb.DescriptorProto, error) {
+	sanitizeMessageType := outputType[strings.Index(outputType, ":")+1:]
+	for _, protoFile := range pkg.ProtoFiles {
+		packageName := protoFile.GetPackage()
+		for _, message := range protoFile.MessageType {
+			if packageName+"."+message.GetName() == sanitizeMessageType {
+				return message, nil
+			}
+
+			nestedMessage := checkNestedMessages(message, packageName, sanitizeMessageType)
+			if nestedMessage != nil {
+				return nestedMessage, nil
+			}
+		}
+	}
+
+	return nil, fmt.Errorf("message type %q not found in package", sanitizeMessageType)
+}
+
+func checkNestedMessages(message *descriptorpb.DescriptorProto, packageName, messageType string) *descriptorpb.DescriptorProto {
+	for _, nestedMessage := range message.NestedType {
+		if packageName+"."+message.GetName()+"."+nestedMessage.GetName() == messageType {
+			return nestedMessage
+		}
+
+		checkNestedMessages(nestedMessage, packageName, messageType)
+	}
+
+	return nil
+}
+
+func getExistingProtoTypes(protoFiles []*descriptorpb.FileDescriptorProto) map[string]*descriptorpb.DescriptorProto {
+	var protoTypeMapping = map[string]*descriptorpb.DescriptorProto{}
+	for _, protoFile := range protoFiles {
+		packageName := protoFile.GetPackage()
+		for _, message := range protoFile.MessageType {
+			currentName := "." + packageName + "." + message.GetName()
+			protoTypeMapping[currentName] = message
+			processMessage(message, currentName, protoTypeMapping)
+		}
+	}
+
+	return protoTypeMapping
+}
+
+func processMessage(message *descriptorpb.DescriptorProto, parentName string, protoTypeMapping map[string]*descriptorpb.DescriptorProto) {
+	for _, nestedMessage := range message.NestedType {
+		currentName := parentName + "." + nestedMessage.GetName()
+		protoTypeMapping[currentName] = nestedMessage
+		processMessage(nestedMessage, currentName, protoTypeMapping)
+	}
+}
+
+func buildGenerateCommandFromArgs(manifestPath, outputType string, withDevEnv bool) error {
+	_, err := os.Stat(".devcontainer")
+
+	isInDevContainer := !os.IsNotExist(err)
+
+	reader, err := manifest.NewReader(manifestPath)
+	if err != nil {
+		return fmt.Errorf("manifest reader: %w", err)
+	}
+
+	pkg, _, err := reader.Read()
+	if err != nil {
+		return fmt.Errorf("read manifest %q: %w", manifestPath, err)
+	}
+
+	moduleNames := []string{}
+	for _, module := range pkg.Modules.Modules {
+		if module.Output != nil {
+			moduleNames = append(moduleNames, module.Name)
+		}
+	}
+
+	selectedModule, err := createRequestModuleForm(moduleNames)
+	if err != nil {
+		return fmt.Errorf("creating request module form: %w", err)
+	}
+
+	requestedModule, err := getModule(pkg, selectedModule)
+	if err != nil {
+		return fmt.Errorf("getting module: %w", err)
+	}
+
+	if outputType == outputTypeSQL {
+		if requestedModule.Output.Type != "proto:sf.substreams.sink.database.v1.DatabaseChanges" {
+			return fmt.Errorf("requested module shoud have proto:sf.substreams.sink.database.v1.DatabaseChanges as output type")
+		}
+	}
+
+	if pkg.GetPackageMeta()[0] == nil {
+		return fmt.Errorf("package meta not found")
+	}
+
+	projectName := pkg.GetPackageMeta()[0].Name
+
+	messageDescriptor, err := searchForMessageTypeIntoPackage(pkg, requestedModule.Output.Type)
+	if err != nil {
+		return fmt.Errorf("searching for message type: %w", err)
+	}
+
+	protoTypeMapping := getExistingProtoTypes(pkg.ProtoFiles)
+
+	if pkg.Network == "" {
+		return fmt.Errorf("network not found in your manifest file")
+	}
+
+	project := NewProject(projectName, pkg.Network, requestedModule, messageDescriptor, protoTypeMapping)
+
+	// Create an example entity from the output descriptor
+	project.BuildExampleEntity()
+
+	projectFiles, err := project.Render(outputType, withDevEnv)
+	if err != nil {
+		return fmt.Errorf("rendering project files: %w", err)
+	}
+
+	saveDir := "subgraph"
+	if cwd, err := os.Getwd(); err == nil {
+		saveDir = filepath.Join(cwd, saveDir)
+	}
+
+	if !isInDevContainer {
+		saveDir, err = createSaveDirForm(saveDir)
+		if err != nil {
+			fmt.Println("creating save directory: %w", err)
+		}
+	}
+
+	err = saveProjectFiles(projectFiles, saveDir)
+	if err != nil {
+		fmt.Println("saving project files: %w", err)
+	}
+
+	return nil
+}
+
+func createSaveDirForm(saveDir string) (string, error) {
+	inputField := huh.NewInput().Title("In which directory do you want to generate the project?").Value(&saveDir)
+	var WITH_ACCESSIBLE = false
+
+	err := huh.NewForm(huh.NewGroup(inputField)).WithTheme(huh.ThemeCharm()).WithAccessible(WITH_ACCESSIBLE).Run()
+	if err != nil {
+		return "", fmt.Errorf("failed taking input: %w", err)
+	}
+
+	return saveDir, nil
+}
+
+func saveProjectFiles(projectFiles map[string][]byte, saveDir string) error {
+	err := os.MkdirAll(saveDir, 0755)
+	if err != nil {
+		return fmt.Errorf("creating directory %s: %w", saveDir, err)
+	}
+
+	for fileName, fileContent := range projectFiles {
+		filePath := filepath.Join(saveDir, fileName)
+
+		err := os.MkdirAll(filepath.Dir(filePath), 0755)
+		if err != nil {
+			return fmt.Errorf("creating directory %s: %w", filepath.Dir(filePath), err)
+		}
+
+		err = os.WriteFile(filePath, fileContent, 0644)
+		if err != nil {
+			return fmt.Errorf("saving file %s: %w", filePath, err)
+		}
+	}
+
+	return nil
+}
+
+func createRequestModuleForm(labels []string) (string, error) {
+	if len(labels) == 0 {
+		fmt.Println("Hmm, the server sent no option to select from (!)")
+	}
+
+	var options []huh.Option[string]
+	optionsMap := make(map[string]string)
+	for i := 0; i < len(labels); i++ {
+		entry := huh.Option[string]{
+			Key:   labels[i],
+			Value: labels[i],
+		}
+		options = append(options, entry)
+		optionsMap[entry.Value] = entry.Key
+	}
+	var selection string
+	selectField := huh.NewSelect[string]().
+		Options(options...).
+		Value(&selection)
+
+	err := huh.NewForm(huh.NewGroup(selectField)).WithTheme(huh.ThemeCharm()).Run()
+	if err != nil {
+		return "", fmt.Errorf("failed taking input: %w", err)
+	}
+
+	return selection, nil
+}