diff --git a/cmd/idea.go b/cmd/idea.go index 33efc97..c62113d 100644 --- a/cmd/idea.go +++ b/cmd/idea.go @@ -1,11 +1,14 @@ package cmd import ( + "fmt" "github.com/Graylog2/graylog-project-cli/config" "github.com/Graylog2/graylog-project-cli/idea" "github.com/Graylog2/graylog-project-cli/manifest" p "github.com/Graylog2/graylog-project-cli/project" + "github.com/Graylog2/graylog-project-cli/utils" "github.com/spf13/cobra" + "github.com/spf13/viper" ) var ideaCmd = &cobra.Command{ @@ -17,8 +20,9 @@ Commands to help working with the IntelliJ IDE. } var ideaSetupCmd = &cobra.Command{ - Use: "setup", - Short: "Setup IntelliJ IDEA", + Deprecated: `use the "run-config create" command instead`, + Use: "setup", + Short: "Setup IntelliJ IDEA", Long: ` This command will do the following: @@ -30,8 +34,47 @@ This command will do the following: Run: ideaSetupCommand, } +var ideaRunConfigsCmd = &cobra.Command{ + Use: "run-configs", + Aliases: []string{"rc"}, + Short: "Manage IntelliJ IDEA run configurations", +} + +var ideaRunConfigsCreateCmd = &cobra.Command{ + Use: "create", + Aliases: []string{"c"}, + Short: "Create IntelliJ IDEA run configurations", + Long: `This command adds default IntelliJ run configurations for Graylog Server, +Data Node, and the web development server. + +The run configurations are created in the $PWD/.run/ directory. + +Examples: + # Create default run configurations + graylog-project idea run-configs create + + # Create default run configurations and .env files (requires installation of EnvFile plugin in IntelliJ) + graylog-project idea run-configs create -E + + # Create run configurations for two Server and three Data Node instances + graylog-project idea run-configs create --instances server=2,data-node=3 +`, + RunE: ideaRunConfigCreateCommand, +} + func init() { + ideaRunConfigsCreateCmd.Flags().BoolP("force", "f", false, "Overwrite existing run configurations") + ideaRunConfigsCreateCmd.Flags().BoolP("env-file", "E", false, "Use .env files (requires the IntelliJ EnvFile plugin)") + ideaRunConfigsCreateCmd.Flags().StringToIntP("instances", "i", idea.DefaultInstanceCounts, "Number of instances - example: server=1,data-node=3") + ideaRunConfigsCreateCmd.Flags().String("root-password", idea.DefaultRootPassword, "The root user password") + ideaRunConfigsCmd.AddCommand(ideaRunConfigsCreateCmd) + + if err := viper.BindPFlags(ideaRunConfigsCreateCmd.Flags()); err != nil { + panic(err) + } + ideaCmd.AddCommand(ideaSetupCmd) + ideaCmd.AddCommand(ideaRunConfigsCmd) RootCmd.AddCommand(ideaCmd) } @@ -41,3 +84,19 @@ func ideaSetupCommand(cmd *cobra.Command, args []string) { idea.Setup(project) } + +func ideaRunConfigCreateCommand(cmd *cobra.Command, args []string) error { + var cfg idea.RunConfig + if err := viper.Unmarshal(&cfg); err != nil { + return fmt.Errorf("couldn't parse configuration: %w", err) + } + + wd, err := utils.GetCwdE() + if err != nil { + return err + } + + cfg.Workdir = wd + + return idea.CreateRunConfigurations(cfg) +} diff --git a/go.mod b/go.mod index 3770d09..8e717ed 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,10 @@ require ( github.com/fatih/color v1.18.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/go-github/v66 v66.0.0 + github.com/google/renameio/v2 v2.0.0 github.com/hashicorp/go-version v1.7.0 github.com/imdario/mergo v0.3.16 - github.com/k0kubun/pp/v3 v3.3.0 + github.com/k0kubun/pp/v3 v3.4.1 github.com/manifoldco/promptui v0.9.0 github.com/mattn/go-isatty v0.0.20 github.com/pelletier/go-toml/v2 v2.2.3 @@ -20,10 +21,12 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + github.com/subosito/gotenv v1.6.0 github.com/yuin/goldmark v1.7.8 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f golang.org/x/oauth2 v0.24.0 golang.org/x/text v0.20.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -45,12 +48,10 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/term v0.26.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) // Workaround for https://github.com/golang/go/issues/30831 diff --git a/go.sum b/go.sum index be192c8..2e49df5 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/google/go-github/v66 v66.0.0 h1:ADJsaXj9UotwdgK8/iFZtv7MLc8E8WBl62WLd github.com/google/go-github/v66 v66.0.0/go.mod h1:+4SO9Zkuyf8ytMj0csN1NR/5OTR+MfqPp8P8dVlcvY4= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -36,8 +38,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/k0kubun/pp/v3 v3.3.0 h1:/Unrck5tDGUSjsUJsmx9GUL64pNKOY5UEdoP1F7FBq8= -github.com/k0kubun/pp/v3 v3.3.0/go.mod h1:wJadGBvcY6JKaiUkB89VzUACKDmTX1r4aQTPERpZc6w= +github.com/k0kubun/pp/v3 v3.4.1 h1:1WdFZDRRqe8UsR61N/2RoOZ3ziTEqgTPVqKrHeb779Y= +github.com/k0kubun/pp/v3 v3.4.1/go.mod h1:+SiNiqKnBfw1Nkj82Lh5bIeKQOAkPy6Xw9CAZUZ8npI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/idea/run_configs.go b/idea/run_configs.go new file mode 100644 index 0000000..71cdb02 --- /dev/null +++ b/idea/run_configs.go @@ -0,0 +1,410 @@ +package idea + +import ( + "bytes" + "crypto/sha256" + "encoding/xml" + "fmt" + "github.com/Graylog2/graylog-project-cli/logger" + "github.com/Graylog2/graylog-project-cli/utils" + "github.com/fatih/color" + "github.com/samber/lo" + "github.com/subosito/gotenv" + "golang.org/x/text/cases" + "golang.org/x/text/language" + "gopkg.in/yaml.v3" + "io/fs" + "maps" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "text/template" +) + +var DefaultRootPassword = "admin" +var DefaultInstanceCounts = map[string]int{ + "server": 2, + "data-node": 2, +} + +var runConfigTemplateDir = filepath.Join(".config", "idea", "templates", "run-configurations") +var configFile = filepath.Join(".config", "idea", "config.yml") + +const runConfigDir = ".run" +const runConfigSuffix = ".run.xml" +const runConfigTemplateSuffix = ".run.xml.template" +const envFileSuffix = ".env.template" +const generatedFilePrefix = "project-generated-" + +// We use a static password secret to ensure that different setups can use the same database. +const staticPasswordSecret = "hCXFTrzZFF88gnVon2fSV6WmAoQANRUqsYFTRbac8WStamVeJkjTXSykWv6FiXDbTYQQnvdTn59iALnkiT6m93BfhDju9Uqh" + +type RunConfig struct { + Workdir string `mapstructure:"workdir"` + Instances map[string]int `mapstructure:"instances"` + Force bool `mapstructure:"force"` + EnvFile bool `mapstructure:"env-file"` + RootPassword string `mapstructure:"root-password"` +} + +type ConfigData struct { + DataDirectories map[string][]string `yaml:"data-directories"` + CompoundConfigs map[string]CompoundConfig `yaml:"compound-configs"` + HostnameEnv []string `yaml:"hostname-env"` +} + +type CompoundConfig struct { + Name string `yaml:"name"` + InstanceTypes []string `yaml:"instance-types"` +} + +type templateData struct { + ConfigName string + InstanceType string + InstanceNumber int + UseEnvFile bool + Env map[string]string + PortOffset int + PasswordSecret string + RootPasswordHash string + DataDir string + IsLeaderNode bool + IsLinux bool + IsDarwin bool + IsWindows bool +} + +var mathTemplateFuncs = map[string]any{ + "add": func(a, b int) int { + return a + b + }, + "sub": func(a, b int) int { + return a - b + }, +} + +// Template string for an IntelliJ compound run configuration entry. +var compoundTemplate = ` + +{{- range .ToRun }} + +{{- end }} + + + +` + +type RunConfigEntry struct { + Name string + InstanceType string + InstanceNumber int + PortOffset int + Template *template.Template + RenderedTemplate bytes.Buffer + EnvTemplate *template.Template + RenderedEnvTemplate bytes.Buffer + DataDirectories []string + Filename string + EnvFilename string + DataDir string +} + +// XMLRunConfig is used to parse the component type out of run configuration files. +type XMLRunConfig struct { + XMLName xml.Name `xml:"component"` + Configuration struct { + Type string `xml:"type,attr"` + } `xml:"configuration"` +} + +type CompoundToRun struct { + Name string + Type string +} + +func CreateRunConfigurations(config RunConfig) error { + rcDir := filepath.Join(config.Workdir, runConfigDir) + tmplDir := filepath.Join(config.Workdir, runConfigTemplateDir) + + if _, err := os.Stat(tmplDir); os.IsNotExist(err) { + return fmt.Errorf("template directory %q doesn't exist; update your repository", tmplDir) + } + + if err := os.MkdirAll(rcDir, 0755); err != nil { + return fmt.Errorf("couldn't create run configurations directory: %w", err) + } + + templates, err := findRunConfigTemplates(tmplDir) + if err != nil { + return err + } + + invalidParams := make([]string, 0) + for name := range config.Instances { + if _, found := templates[name]; !found { + invalidParams = append(invalidParams, name) + } + } + if len(invalidParams) > 0 { + return fmt.Errorf("invalid instance count parameter(s): %s (available: %s)", + strings.Join(invalidParams, ", "), strings.Join(slices.Sorted(maps.Keys(templates)), ", ")) + } + + configData, err := parseConfigFile(filepath.Join(config.Workdir, configFile)) + if err != nil { + return err + } + + entries := make([]RunConfigEntry, 0) + hostnames := make(map[string]bool) + + // Build all run configuration entries in memory + for instanceType, tmpl := range templates { + totalCount := getInstanceCount(config, instanceType) + for i := range totalCount { + num := i + 1 + + envTmpl, err := findEnvFileTemplate(filepath.Join(config.Workdir, runConfigTemplateDir), instanceType) + if err != nil { + return err + } + + entry := RunConfigEntry{ + Name: generateConfigName(instanceType, num, totalCount), + InstanceType: instanceType, + InstanceNumber: num, + PortOffset: i, + Template: tmpl, + EnvTemplate: envTmpl, + DataDirectories: configData.DataDirectories[instanceType], + Filename: fmt.Sprintf("%s%s-%d%s", generatedFilePrefix, instanceType, num, runConfigSuffix), + EnvFilename: fmt.Sprintf(".env.%s-%d", instanceType, num), + DataDir: filepath.Join("data", fmt.Sprintf("%s-%d", instanceType, num)), + } + + data := templateData{ + ConfigName: entry.Name, + InstanceType: entry.InstanceType, + InstanceNumber: entry.InstanceNumber, + PortOffset: entry.PortOffset, + UseEnvFile: config.EnvFile, + PasswordSecret: staticPasswordSecret, + RootPasswordHash: fmt.Sprintf("%x", sha256.Sum256([]byte(config.RootPassword))), + DataDir: entry.DataDir, + IsLeaderNode: entry.InstanceNumber == 1, // Make the first node the leader node + IsLinux: runtime.GOOS == "linux", + IsDarwin: runtime.GOOS == "darwin", + IsWindows: runtime.GOOS == "windows", + } + + if entry.EnvTemplate != nil { + if err := entry.EnvTemplate.Execute(&entry.RenderedEnvTemplate, data); err != nil { + return fmt.Errorf("couldn't render env-file template: %w", err) + } + } + + data.Env = gotenv.Parse(bytes.NewReader(entry.RenderedEnvTemplate.Bytes())) + + if err := entry.Template.Execute(&entry.RenderedTemplate, data); err != nil { + return fmt.Errorf("couldn't compile template: %w", err) + } + + for key, value := range data.Env { + if slices.Contains(configData.HostnameEnv, key) { + hostnames[value] = true + } + } + + entries = append(entries, entry) + } + } + + // Write all run configuration entries to the file system + for _, entry := range entries { + if err := writeEntryFiles(rcDir, config, entry); err != nil { + return err + } + } + + // Write all compound run configurations + for name, cfg := range configData.CompoundConfigs { + compoundFilename := fmt.Sprintf("%scompound-%s%s", generatedFilePrefix, name, runConfigSuffix) + compoundFilepath := filepath.Join(config.Workdir, runConfigDir, compoundFilename) + + if _, err := os.Stat(compoundFilepath); !os.IsNotExist(err) && !config.Force { + logger.Info("Skipping existing compound configuration: %s", filepath.Join(runConfigDir, compoundFilename)) + continue + } + + filteredEntries := lo.Filter(entries, func(entry RunConfigEntry, index int) bool { + return slices.Contains(cfg.InstanceTypes, entry.InstanceType) + }) + + toRun := make([]CompoundToRun, 0) + + for _, entry := range filteredEntries { + var xrc XMLRunConfig + if err := xml.NewDecoder(bytes.NewReader(entry.RenderedTemplate.Bytes())).Decode(&xrc); err != nil { + return fmt.Errorf("couldn't parse rendered run configuration %q: %w", entry.Filename, err) + } + + toRun = append(toRun, CompoundToRun{Name: entry.Name, Type: xrc.Configuration.Type}) + } + + tmpl, err := template.New(cfg.Name).Parse(compoundTemplate) + if err != nil { + return fmt.Errorf("couldn't parse compound template %q: %w", cfg.Name, err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, map[string]any{"Name": cfg.Name, "ToRun": toRun}); err != nil { + return err + } + + if err := utils.AtomicallyWriteFile(compoundFilepath, buf.Bytes(), 0600); err != nil { + return fmt.Errorf("couldn't write compound file %q: %w", compoundFilename, err) + } + + logger.Info("Created compound configuration: %s", filepath.Join(runConfigDir, compoundFilename)) + } + + if len(hostnames) > 0 { + logger.Info("\nThe following template values seem to contain hostnames.") + for _, name := range slices.Sorted(maps.Keys(hostnames)) { + logger.Info(" - %v", name) + } + logger.Info("Please ensure all hostnames resolve correctly!") + if runtime.GOOS == "linux" { + // On Linux with systemd (basically all these days) *.localhost resolves to 127.0.0.1. + logger.ColorInfo(color.FgGreen, "You are on Linux, so the hostnames should work automatically.") + } else { + logger.ColorInfo(color.FgYellow, "You are on %s, please add the hostnames to your /etc/hosts file using 127.0.0.1 as the IP address.", runtime.GOOS) + } + } + + return nil +} + +func parseConfigFile(path string) (*ConfigData, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("couldn't open config %q: %w", path, err) + } + //goland:noinspection ALL + defer f.Close() + + var value ConfigData + if err := yaml.NewDecoder(f).Decode(&value); err != nil { + return nil, fmt.Errorf("couldn't parse idea config %q: %w", path, err) + } + + return &value, nil +} + +func getInstanceCount(config RunConfig, instanceType string) int { + count, ok := config.Instances[instanceType] + if !ok { + defaultCount, defaultOk := DefaultInstanceCounts[instanceType] + if defaultOk { + return defaultCount + } else { + return 1 + } + } + return count +} + +func writeEntryFiles(configDir string, config RunConfig, entry RunConfigEntry) error { + if _, err := os.Stat(filepath.Join(configDir, entry.Filename)); !os.IsNotExist(err) && !config.Force { + logger.Info("Skipping existing run configuration: %s", filepath.Join(runConfigDir, entry.Filename)) + return nil + } + + if err := utils.AtomicallyWriteFile(filepath.Join(configDir, entry.Filename), entry.RenderedTemplate.Bytes(), 0600); err != nil { + return fmt.Errorf("couldn't write file %q: %w", entry.Filename, err) + } + + logger.Info("Created run configuration: %s", filepath.Join(runConfigDir, entry.Filename)) + + for _, dir := range entry.DataDirectories { + dirToCreate := filepath.Join(entry.DataDir, dir) + if _, err := os.Stat(dirToCreate); os.IsNotExist(err) { + if err := os.MkdirAll(dirToCreate, 0755); err != nil { + return fmt.Errorf("couldn't create data dir %q: %w", dir, err) + } + logger.Info("Created data directory: %s", dirToCreate) + } + } + + if config.EnvFile { + if err := utils.AtomicallyWriteFile(filepath.Join(config.Workdir, entry.EnvFilename), entry.RenderedEnvTemplate.Bytes(), 0600); err != nil { + return fmt.Errorf("couldn't write file %q: %w", entry.EnvFilename, err) + } + + logger.Info("Created run env file: %s", entry.EnvFilename) + } + + return nil +} + +func generateConfigName(instanceType string, num int, total int) string { + // Data-Node -> Data Node + configName := strings.ReplaceAll(cases.Title(language.English).String(instanceType), "-", " ") + + if total > 1 { + return fmt.Sprintf("%s %d", configName, num) + } else { + return configName + } +} + +func findEnvFileTemplate(templateDir string, name string) (*template.Template, error) { + data, err := os.ReadFile(filepath.Join(templateDir, fmt.Sprintf("%s%s", name, envFileSuffix))) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("couldn't read env-file template for %q: %w", name, err) + } + tmpl, err := template.New(name).Option("missingkey=error").Funcs(mathTemplateFuncs).Parse(string(data)) + if err != nil { + return nil, fmt.Errorf("couldn't parse template %q: %w", name, err) + } + return tmpl, nil +} + +func findRunConfigTemplates(templateDir string) (map[string]*template.Template, error) { + templates := make(map[string]*template.Template) + + err := filepath.WalkDir(templateDir, func(path string, entry fs.DirEntry, err error) error { + if entry.IsDir() { + return nil + } + if !strings.HasSuffix(entry.Name(), runConfigTemplateSuffix) { + return nil + } + + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("couldn't read template: %w", err) + } + + name := strings.TrimSuffix(entry.Name(), runConfigTemplateSuffix) + + tmpl, err := template.New(name).Option("missingkey=error").Funcs(mathTemplateFuncs).Parse(string(data)) + if err != nil { + return fmt.Errorf("couldn't parse template %q: %w", path, err) + } + + templates[name] = tmpl + + return nil + }) + if err != nil { + return nil, fmt.Errorf("couldn't walk template dir: %w", err) + } + + return templates, nil +} diff --git a/idea/run_configs_test.go b/idea/run_configs_test.go new file mode 100644 index 0000000..b7abec8 --- /dev/null +++ b/idea/run_configs_test.go @@ -0,0 +1,275 @@ +package idea + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestCreateRunConfigurations(t *testing.T) { + workdir := t.TempDir() + + config := RunConfig{ + Workdir: workdir, + Instances: DefaultInstanceCounts, + Force: false, + EnvFile: false, + RootPassword: "test", + } + + // No template files yet, should fail + require.ErrorContains(t, CreateRunConfigurations(config), "update your repository") + + // Copy the templates + require.NoError(t, copyDir("testdata", workdir)) + + // Should not fail now + require.NoError(t, CreateRunConfigurations(config)) + + assert.DirExists(t, filepath.Join(workdir, ".run")) + + for _, instance := range []string{ + "compound-all", "compound-data-nodes", "compound-servers", + "data-node-1", "data-node-2", "server-1", "server-2", "web-1", + } { + assert.FileExists(t, filepath.Join(workdir, ".run", "project-generated-"+instance+".run.xml")) + } + + for _, file := range []string{"data-node-1", "data-node-2", "server-1", "server-2", "web-1"} { + assert.NoFileExists(t, filepath.Join(workdir, ".env."+file)) + } + + assertCompoundRunConfigFile(t, filepath.Join(workdir, ".run", "project-generated-compound-all.run.xml"), "All Nodes", map[string]string{ + "Server 1": "Application", + "Server 2": "Application", + "Data Node 1": "Application", + "Data Node 2": "Application", + "Web": "js.build_tools.npm", + }) + assertCompoundRunConfigFile(t, filepath.Join(workdir, ".run", "project-generated-compound-servers.run.xml"), "Servers", map[string]string{ + "Server 1": "Application", + "Server 2": "Application", + }) + assertCompoundRunConfigFile(t, filepath.Join(workdir, ".run", "project-generated-compound-data-nodes.run.xml"), "Data Nodes", map[string]string{ + "Data Node 1": "Application", + "Data Node 2": "Application", + }) + + assertFile(t, assert.Contains, filepath.Join(workdir, ".run", "project-generated-web-1.run.xml"), + ``) + + for _, instance := range []string{"server-1", "server-2"} { + path := filepath.Join(workdir, ".run", fmt.Sprintf("project-generated-%s.run.xml", instance)) + num := getInstanceNumber(t, instance) + portOffset := num - 1 + + assertFile(t, assert.Contains, path, fmt.Sprintf(``, num)) + assertFile(t, assert.NotContains, path, fmt.Sprintf(`PATH="$PROJECT_DIR$/.env.%s`, instance)) + + assertRunConfigFileEnv(t, assert.Contains, path, map[string]any{ + "GRAYLOG_NODE_ID_FILE": filepath.Join("data", instance, "node-id"), + "GRAYLOG_DATA_DIR": filepath.Join("data", instance), + "GRAYLOG_MESSAGE_JOURNAL_DIR": filepath.Join("data", instance, "journal"), + "GRAYLOG_IS_LEADER": num == 1, // First node should be the leader + "GRAYLOG_HTTP_BIND_ADDRESS": fmt.Sprintf("127.0.0.1:%d", 9000+portOffset), + "GRAYLOG_PASSWORD_SECRET": "hCXFTrzZFF88gnVon2fSV6WmAoQANRUqsYFTRbac8WStamVeJkjTXSykWv6FiXDbTYQQnvdTn59iALnkiT6m93BfhDju9Uqh", + "GRAYLOG_ROOT_PASSWORD_SHA2": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + }) + } + for _, instance := range []string{"data-node-1", "data-node-2"} { + path := filepath.Join(workdir, ".run", fmt.Sprintf("project-generated-%s.run.xml", instance)) + num := getInstanceNumber(t, instance) + portOffset := num - 1 + + assertFile(t, assert.Contains, path, fmt.Sprintf(``, num)) + assertFile(t, assert.NotContains, path, fmt.Sprintf(`PATH="$PROJECT_DIR$/.env.%s`, instance)) + + assertRunConfigFileEnv(t, assert.Contains, path, map[string]any{ + "GRAYLOG_DATANODE_PASSWORD_SECRET": "hCXFTrzZFF88gnVon2fSV6WmAoQANRUqsYFTRbac8WStamVeJkjTXSykWv6FiXDbTYQQnvdTn59iALnkiT6m93BfhDju9Uqh", + "GRAYLOG_DATANODE_NODE_ID_FILE": filepath.Join("data", instance, "node-id"), + "GRAYLOG_DATANODE_CONFIG_LOCATION": filepath.Join("data", instance, "config"), + "GRAYLOG_DATANODE_NATIVE_LIB_DIR": filepath.Join("data", instance, "native_libs"), + "GRAYLOG_DATANODE_DATANODE_HTTP_PORT": 8999 - portOffset, + "GRAYLOG_DATANODE_OPENSEARCH_HTTP_PORT": 9200 + portOffset, + "GRAYLOG_DATANODE_OPENSEARCH_TRANSPORT_PORT": 9300 + portOffset, + }) + } +} + +func TestCreateRunConfigurationsWithEnv(t *testing.T) { + workdir := t.TempDir() + + config := RunConfig{ + Workdir: workdir, + Instances: DefaultInstanceCounts, + Force: false, + EnvFile: true, + RootPassword: "test", + } + + // No template files yet, should fail + require.ErrorContains(t, CreateRunConfigurations(config), "update your repository") + + // Copy the templates + require.NoError(t, copyDir("testdata", workdir)) + + // Should not fail now + require.NoError(t, CreateRunConfigurations(config)) + + assert.DirExists(t, filepath.Join(workdir, ".run")) + + for _, file := range []string{ + "compound-all", "compound-data-nodes", "compound-servers", + "data-node-1", "data-node-2", "server-1", "server-2", "web-1", + } { + assert.FileExists(t, filepath.Join(workdir, ".run", "project-generated-"+file+".run.xml")) + } + + for _, file := range []string{"data-node-1", "data-node-2", "server-1", "server-2", "web-1"} { + assert.FileExists(t, filepath.Join(workdir, ".env."+file)) + } + + assertCompoundRunConfigFile(t, filepath.Join(workdir, ".run", "project-generated-compound-all.run.xml"), "All Nodes", map[string]string{ + "Server 1": "Application", + "Server 2": "Application", + "Data Node 1": "Application", + "Data Node 2": "Application", + "Web": "js.build_tools.npm", + }) + assertCompoundRunConfigFile(t, filepath.Join(workdir, ".run", "project-generated-compound-servers.run.xml"), "Servers", map[string]string{ + "Server 1": "Application", + "Server 2": "Application", + }) + assertCompoundRunConfigFile(t, filepath.Join(workdir, ".run", "project-generated-compound-data-nodes.run.xml"), "Data Nodes", map[string]string{ + "Data Node 1": "Application", + "Data Node 2": "Application", + }) + + assertFile(t, assert.Contains, filepath.Join(workdir, ".run", "project-generated-web-1.run.xml"), + ``) + + for _, instance := range []string{"server-1", "server-2"} { + path := filepath.Join(workdir, ".run", fmt.Sprintf("project-generated-%s.run.xml", instance)) + envPath := filepath.Join(workdir, fmt.Sprintf(".env.%s", instance)) + num := getInstanceNumber(t, instance) + portOffset := num - 1 + + assertFile(t, assert.Contains, path, fmt.Sprintf(``, num)) + assertFile(t, assert.Contains, path, fmt.Sprintf(`PATH="$PROJECT_DIR$/.env.%s`, instance)) + + expectedData := map[string]any{ + "GRAYLOG_NODE_ID_FILE": filepath.Join("data", instance, "node-id"), + "GRAYLOG_DATA_DIR": filepath.Join("data", instance), + "GRAYLOG_MESSAGE_JOURNAL_DIR": filepath.Join("data", instance, "journal"), + "GRAYLOG_IS_LEADER": num == 1, // First node should be the leader + "GRAYLOG_HTTP_BIND_ADDRESS": fmt.Sprintf("127.0.0.1:%d", 9000+portOffset), + "GRAYLOG_PASSWORD_SECRET": "hCXFTrzZFF88gnVon2fSV6WmAoQANRUqsYFTRbac8WStamVeJkjTXSykWv6FiXDbTYQQnvdTn59iALnkiT6m93BfhDju9Uqh", + "GRAYLOG_ROOT_PASSWORD_SHA2": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + } + assertRunConfigFileEnv(t, assert.NotContains, path, expectedData) + assertEnvFile(t, assert.Contains, envPath, expectedData) + } + for _, instance := range []string{"data-node-1", "data-node-2"} { + path := filepath.Join(workdir, ".run", fmt.Sprintf("project-generated-%s.run.xml", instance)) + envPath := filepath.Join(workdir, fmt.Sprintf(".env.%s", instance)) + num := getInstanceNumber(t, instance) + portOffset := num - 1 + + assertFile(t, assert.Contains, path, fmt.Sprintf(``, num)) + assertFile(t, assert.Contains, path, fmt.Sprintf(`PATH="$PROJECT_DIR$/.env.%s`, instance)) + + expectedData := map[string]any{ + "GRAYLOG_DATANODE_NODE_ID_FILE": filepath.Join("data", instance, "node-id"), + "GRAYLOG_DATANODE_CONFIG_LOCATION": filepath.Join("data", instance, "config"), + "GRAYLOG_DATANODE_NATIVE_LIB_DIR": filepath.Join("data", instance, "native_libs"), + "GRAYLOG_DATANODE_DATANODE_HTTP_PORT": 8999 - portOffset, + "GRAYLOG_DATANODE_OPENSEARCH_HTTP_PORT": 9200 + portOffset, + "GRAYLOG_DATANODE_OPENSEARCH_TRANSPORT_PORT": 9300 + portOffset, + } + assertRunConfigFileEnv(t, assert.NotContains, path, expectedData) + assertEnvFile(t, assert.Contains, envPath, expectedData) + } +} + +func getInstanceNumber(t *testing.T, name string) int { + parts := strings.SplitAfter(name, "-") + num, err := strconv.Atoi(parts[len(parts)-1]) + require.NoError(t, err) + return num +} + +func assertCompoundRunConfigFile(t *testing.T, path string, title string, values map[string]string) { + assertFile(t, assert.Contains, path, fmt.Sprintf(``, title), "wrong title") + + for key, value := range values { + assertFile(t, assert.Contains, path, fmt.Sprintf(``, key, value), fmt.Sprintf("compound file %q - %s=%s", path, key, value)) + } +} + +func assertRunConfigFileEnv(t *testing.T, check func(assert.TestingT, interface{}, interface{}, ...interface{}) bool, path string, values map[string]any) { + for key, value := range values { + assertFile(t, check, path, fmt.Sprintf(``, key, value), fmt.Sprintf("run config file %q - %s=%s", path, key, value)) + } +} + +func assertEnvFile(t *testing.T, check func(assert.TestingT, interface{}, interface{}, ...interface{}) bool, path string, values map[string]any) { + for key, value := range values { + assertFile(t, check, path, fmt.Sprintf("%s=%v", key, value), fmt.Sprintf("env file %q - %s=%s", path, key, value)) + } +} + +func assertFile(t *testing.T, check func(assert.TestingT, interface{}, interface{}, ...interface{}) bool, path string, needle string, message ...string) { + require.FileExists(t, path) + buf, err := os.ReadFile(path) + require.NoError(t, err) + check(t, string(buf), needle, message) +} + +func copyDir(srcPath, dstPath string) error { + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if path == srcPath { + return nil + } + + dstFilePath := filepath.Join(dstPath, path[len(srcPath):]) + + if info.IsDir() { + if err := os.MkdirAll(dstFilePath, info.Mode()); err != nil { + return err + } + return nil + } + + srcFile, err := os.Open(path) + if err != nil { + return err + } + //goland:noinspection ALL + defer srcFile.Close() + + dstFile, err := os.Create(dstFilePath) + if err != nil { + return err + } + //goland:noinspection ALL + defer dstFile.Close() + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return err + } + + return os.Chmod(dstFilePath, info.Mode()) + } + + return filepath.Walk(srcPath, walkFn) +} diff --git a/idea/testdata/.config/idea/config.yml b/idea/testdata/.config/idea/config.yml new file mode 100644 index 0000000..c1467c4 --- /dev/null +++ b/idea/testdata/.config/idea/config.yml @@ -0,0 +1,41 @@ +--- +# Defines the list of data directories for each node type that should be +# created by the "graylog-project idea run-config create" command. +data-directories: + server: + - "journal" + + data-node: + - "config" + - "logs" + - "opensearch-config" + - "opensearch-data" + +# Environment variables that might contain hostnames which must be configured correctly. +# This is used by the CLI to notify users about setting up the hostnames. (e.g., adding them to /etc/hosts) +hostname-env: + - "GRAYLOG_HTTP_PUBLISH_URI" + - "GRAYLOG_HTTP_EXTERNAL_URI" + - "GRAYLOG_HTTP_BIND_ADDRESS" + - "GRAYLOG_DATANODE_NODE_NAME" + - "GRAYLOG_DATANODE_HOSTNAME" + - "GRAYLOG_DATANODE_BIND_ADDRESS" + +# Describes all comound configs that should be created. +compound-configs: + servers: + name: "Servers" + instance-types: + - "server" + + data-nodes: + name: "Data Nodes" + instance-types: + - "data-node" + + all: + name: "All Nodes" + instance-types: + - "server" + - "data-node" + - "web" diff --git a/idea/testdata/.config/idea/templates/run-configurations/data-node.env.template b/idea/testdata/.config/idea/templates/run-configurations/data-node.env.template new file mode 100644 index 0000000..fbbf894 --- /dev/null +++ b/idea/testdata/.config/idea/templates/run-configurations/data-node.env.template @@ -0,0 +1,20 @@ +# vim: ft=sh +GRAYLOG_DATANODE_PASSWORD_SECRET={{ .PasswordSecret }} +GRAYLOG_DATANODE_NODE_ID_FILE={{ .DataDir }}/node-id +GRAYLOG_DATANODE_CONFIG_LOCATION={{ .DataDir }}/config +GRAYLOG_DATANODE_NATIVE_LIB_DIR={{ .DataDir }}/native_libs + +GRAYLOG_DATANODE_NODE_NAME={{ .InstanceType }}-{{ .InstanceNumber }}.graylog.localhost +GRAYLOG_DATANODE_HOSTNAME={{ .InstanceType }}-{{ .InstanceNumber }}.graylog.localhost +GRAYLOG_DATANODE_BIND_ADDRESS={{ .InstanceType }}-{{ .InstanceNumber }}.graylog.localhost + +GRAYLOG_DATANODE_DATANODE_HTTP_PORT={{ sub 8999 .PortOffset }} +GRAYLOG_DATANODE_OPENSEARCH_HTTP_PORT={{ add 9200 .PortOffset }} +GRAYLOG_DATANODE_OPENSEARCH_TRANSPORT_PORT={{ add 9300 .PortOffset }} +GRAYLOG_DATANODE_OPENSEARCH_HEAP=512m +GRAYLOG_DATANODE_OPENSEARCH_LOCATION=../graylog-project-repos/graylog2-server/data-node/target/opensearch +GRAYLOG_DATANODE_OPENSEARCH_LOGS_LOCATION={{ .DataDir }}/logs +GRAYLOG_DATANODE_OPENSEARCH_CONFIG_LOCATION={{ .DataDir }}/opensearch-config +GRAYLOG_DATANODE_OPENSEARCH_DATA_LOCATION={{ .DataDir }}/opensearch-data + +#GRAYLOG_DATANODE_INSECURE_STARTUP=true diff --git a/idea/testdata/.config/idea/templates/run-configurations/data-node.run.xml.template b/idea/testdata/.config/idea/templates/run-configurations/data-node.run.xml.template new file mode 100644 index 0000000..5ad7f67 --- /dev/null +++ b/idea/testdata/.config/idea/templates/run-configurations/data-node.run.xml.template @@ -0,0 +1,33 @@ + + + + +{{- if not .UseEnvFile }} + {{- range $key, $value := .Env }} + + {{- end }} +{{- end }} + + + diff --git a/idea/testdata/.config/idea/templates/run-configurations/server.env.template b/idea/testdata/.config/idea/templates/run-configurations/server.env.template new file mode 100644 index 0000000..38d604a --- /dev/null +++ b/idea/testdata/.config/idea/templates/run-configurations/server.env.template @@ -0,0 +1,17 @@ +# vim: ft=sh +GRAYLOG_NODE_ID_FILE={{ .DataDir }}/node-id +# Required to pass the pre-flight check for the Enterprise binaries. +GRAYLOG_BIN_DIR=../graylog-project-repos/graylog-plugin-enterprise/enterprise/bin +GRAYLOG_DATA_DIR={{ .DataDir }} +GRAYLOG_MESSAGE_JOURNAL_DIR={{ .DataDir }}/journal +GRAYLOG_PASSWORD_SECRET={{ .PasswordSecret }} +GRAYLOG_ROOT_PASSWORD_SHA2={{ .RootPasswordHash }} +GRAYLOG_HTTP_BIND_ADDRESS=127.0.0.1:{{ add 9000 .PortOffset }} +GRAYLOG_IS_LEADER={{ .IsLeaderNode }} +#GRAYLOG_LEADER_ELECTION_MODE=automatic +GRAYLOG_LB_RECOGNITION_PERIOD_SECONDS=0 +GRAYLOG_VERSIONCHECKS=false +GRAYLOG_TELEMETRY_ENABLED=false + +GRAYLOG_MONGODB_URI=mongodb://127.0.0.1:27017/graylog +#GRAYLOG_ELASTICSEARCH_HOSTS=http://127.0.0.1:9200,http://127.0.0.1:9201 diff --git a/idea/testdata/.config/idea/templates/run-configurations/server.run.xml.template b/idea/testdata/.config/idea/templates/run-configurations/server.run.xml.template new file mode 100644 index 0000000..ffd81de --- /dev/null +++ b/idea/testdata/.config/idea/templates/run-configurations/server.run.xml.template @@ -0,0 +1,33 @@ + + + + +{{- if not .UseEnvFile }} + {{- range $key, $value := .Env }} + + {{- end }} +{{- end }} + + + diff --git a/idea/testdata/.config/idea/templates/run-configurations/web.run.xml.template b/idea/testdata/.config/idea/templates/run-configurations/web.run.xml.template new file mode 100644 index 0000000..44d169c --- /dev/null +++ b/idea/testdata/.config/idea/templates/run-configurations/web.run.xml.template @@ -0,0 +1,15 @@ + + + + + +