Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added out of order file sorting #482

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 176 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# Created by https://www.toptal.com/developers/gitignore/api/go,terraform,intellij
# Edit at https://www.toptal.com/developers/gitignore?templates=go,terraform,intellij

### Go ###
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib

# Test binary, built with `go test -c`
*.test

# Output of the go coverage tool, specifically when used with LiteIDE
*.out

# Dependency directories (remove the comment below to include it)
# vendor/

# Go workspace file
go.work

### Intellij ###
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839

# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf

# AWS User-specific
.idea/**/aws.xml

# Generated files
.idea/**/contentModel.xml

# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml

# Gradle
.idea/**/gradle.xml
.idea/**/libraries

# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr

# CMake
cmake-build-*/

# Mongo Explorer plugin
.idea/**/mongoSettings.xml

# File-based project format
*.iws

# IntelliJ
out/

# mpeltonen/sbt-idea plugin
.idea_modules/

# JIRA plugin
atlassian-ide-plugin.xml

# Cursive Clojure plugin
.idea/replstate.xml

# SonarLint plugin
.idea/sonarlint/

# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties

# Editor-based Rest Client
.idea/httpRequests

# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser

### Intellij Patch ###
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721

# *.iml
# modules.xml
# .idea/misc.xml
# *.ipr

# Sonarlint plugin
# https://plugins.jetbrains.com/plugin/7973-sonarlint
.idea/**/sonarlint/

# SonarQube Plugin
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
.idea/**/sonarIssues.xml

# Markdown Navigator plugin
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
.idea/**/markdown-navigator.xml
.idea/**/markdown-navigator-enh.xml
.idea/**/markdown-navigator/

# Cache file creation bug
# See https://youtrack.jetbrains.com/issue/JBR-2257
.idea/$CACHE_FILE$

# CodeStream plugin
# https://plugins.jetbrains.com/plugin/12206-codestream
.idea/codestream.xml

# Azure Toolkit for IntelliJ plugin
# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
.idea/**/azureSettings.xml

### Terraform ###
# Local .terraform directories
**/.terraform/*

# .tfstate files
*.tfstate
*.tfstate.*

# Crash log files
crash.log
crash.*.log

# Exclude all .tfvars files, which are likely to contain sensitive data, such as
# password, private keys, and other secrets. These should not be part of version
# control as they are data points which are potentially sensitive and subject
# to change depending on the environment.
*.tfvars
*.tfvars.json

# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json

# Include override files you do wish to add to version control using negated pattern
# !example_override.tf

# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*

# Ignore CLI configuration files
.terraformrc
terraform.rc

# End of https://www.toptal.com/developers/gitignore/api/go,terraform,intellij
2 changes: 1 addition & 1 deletion avro_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package avro_test

import "github.com/hamba/avro/v2"
import "github.com/justtrackio/avro/v2"

func ConfigTeardown() {
// Reset the caches
Expand Down
2 changes: 1 addition & 1 deletion bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"os"
"testing"

"github.com/hamba/avro/v2"
"github.com/justtrackio/avro/v2"
)

type Superhero struct {
Expand Down
176 changes: 173 additions & 3 deletions cmd/avrogen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package main

import (
"bytes"
"container/heap"
"encoding/json"
"errors"
"flag"
"fmt"
Expand All @@ -10,8 +12,8 @@ import (
"path/filepath"
"strings"

"github.com/hamba/avro/v2"
"github.com/hamba/avro/v2/gen"
"github.com/justtrackio/avro/v2"
"github.com/justtrackio/avro/v2/gen"
"golang.org/x/tools/imports"
)

Expand Down Expand Up @@ -89,7 +91,14 @@ func realMain(args []string, stdout, stderr io.Writer) int {
gen.WithFullSchema(cfg.FullSchema),
}
g := gen.NewGenerator(cfg.Pkg, tags, opts...)
for _, file := range flgs.Args() {

files, err := sortFiles(flgs.Args())
if err != nil {
_, _ = fmt.Fprintf(stderr, "Error: %v\n", err)
return 2
}

for _, file := range files {
schema, err := avro.ParseFiles(filepath.Clean(file))
if err != nil {
_, _ = fmt.Fprintf(stderr, "Error: %v\n", err)
Expand Down Expand Up @@ -118,6 +127,167 @@ func realMain(args []string, stdout, stderr io.Writer) int {
return 0
}

type doc struct {
Type avro.Type
Name string
Namespace string
Fields []field
}

type field struct {
Name string
Type any
}

type graph map[string]map[string]struct{}

func sortFiles(args []string) ([]string, error) {
deps := make(graph)

paths := make(map[string]string)
for _, filePath := range args {
file, err := os.Open(filepath.Clean(filePath))
if err != nil {
return nil, err
}

asBytes, err := io.ReadAll(file)
if err != nil {
return nil, err
}

var doc doc
if err := json.Unmarshal(asBytes, &doc); err != nil {
return nil, err
}

name := fmt.Sprintf("%s.%s", doc.Namespace, doc.Name)
paths[name] = filePath

if _, ok := deps[name]; !ok {
deps[name] = make(map[string]struct{})
}

for _, f := range doc.Fields {
for _, dep := range getDependencies(f.Type) {
if !strings.Contains(dep, ".") {
dep = fmt.Sprintf("%s.%s", doc.Namespace, dep)
}

if _, ok := deps[dep]; !ok {
deps[dep] = make(map[string]struct{})
}

deps[dep][name] = struct{}{}
}
}
}

indegree := make(map[string]int)
for node := range deps {
indegree[node] = 0
}

for _, neighbours := range deps {
for neighbor := range neighbours {
indegree[neighbor]++
}
}

sorted, err := kahnTopologicSort(deps, indegree)
if err != nil {
return nil, err
}

var sortedPaths []string
for _, it := range sorted {
if path, ok := paths[it]; ok {
sortedPaths = append(sortedPaths, path)
} else {
return nil, fmt.Errorf("could not find path for %s", it)
}
}

return sortedPaths, nil
}

type MinHeap []string

func (h MinHeap) Len() int { return len(h) }
func (h MinHeap) Less(i, j int) bool { return h[i] < h[j] }
func (h MinHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *MinHeap) Push(x any) {
*h = append(*h, x.(string))
}

func (h *MinHeap) Pop() any {
old := *h
n := len(old)
x := old[n-1]
*h = old[:n-1]
return x
}

func kahnTopologicSort(deps graph, indegree map[string]int) ([]string, error) {
queue := &MinHeap{}
heap.Init(queue)

var result []string

for node, deg := range indegree {
if deg == 0 {
heap.Push(queue, node)
}
}

for queue.Len() > 0 {
front := heap.Pop(queue)
node := front.(string)
result = append(result, node)

for neighbor := range deps[node] {
indegree[neighbor]--
if indegree[neighbor] == 0 {
heap.Push(queue, neighbor)
}
}
}

if len(result) != len(deps) {
return nil, fmt.Errorf("could not sort files, since they contain a cyclic dependency, or files are missing")
}

return result, nil
}

func isBuildIn(t string) bool {
switch avro.Type(t) {
case avro.String, avro.Bytes, avro.Fixed, avro.Int, avro.Long, avro.Double, avro.Boolean, avro.Null, avro.Float, avro.Array, avro.Map:
return true
}

return false
}

func getDependencies(t any) (deps []string) {
switch val := t.(type) {
case nil:
return deps
case string:
if !isBuildIn(val) {
deps = append(deps, val)
}
case []any:
for _, it := range val {
deps = append(deps, getDependencies(it)...)
}
case map[string]any:
deps = append(deps, getDependencies(val["type"])...)
}

return
}

func writeOut(filename string, stdout io.Writer, bytes []byte) error {
writer := stdout
if filename != "" {
Expand Down
Loading