Skip to content

Commit

Permalink
added out of order file sorting
Browse files Browse the repository at this point in the history
  • Loading branch information
nikplxjt committed Dec 13, 2024
1 parent 45d131b commit bd4efff
Show file tree
Hide file tree
Showing 63 changed files with 751 additions and 67 deletions.
176 changes: 176 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# Created by https://www.toptal.com/developers/gitignore/api/go,terraform,intellij
# Edit at https://www.toptal.com/developers/gitignore?templates=go,terraform,intellij

### Go ###
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib

# Test binary, built with `go test -c`
*.test

# Output of the go coverage tool, specifically when used with LiteIDE
*.out

# Dependency directories (remove the comment below to include it)
# vendor/

# Go workspace file
go.work

### Intellij ###
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839

# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf

# AWS User-specific
.idea/**/aws.xml

# Generated files
.idea/**/contentModel.xml

# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml

# Gradle
.idea/**/gradle.xml
.idea/**/libraries

# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr

# CMake
cmake-build-*/

# Mongo Explorer plugin
.idea/**/mongoSettings.xml

# File-based project format
*.iws

# IntelliJ
out/

# mpeltonen/sbt-idea plugin
.idea_modules/

# JIRA plugin
atlassian-ide-plugin.xml

# Cursive Clojure plugin
.idea/replstate.xml

# SonarLint plugin
.idea/sonarlint/

# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties

# Editor-based Rest Client
.idea/httpRequests

# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser

### Intellij Patch ###
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721

# *.iml
# modules.xml
# .idea/misc.xml
# *.ipr

# Sonarlint plugin
# https://plugins.jetbrains.com/plugin/7973-sonarlint
.idea/**/sonarlint/

# SonarQube Plugin
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
.idea/**/sonarIssues.xml

# Markdown Navigator plugin
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
.idea/**/markdown-navigator.xml
.idea/**/markdown-navigator-enh.xml
.idea/**/markdown-navigator/

# Cache file creation bug
# See https://youtrack.jetbrains.com/issue/JBR-2257
.idea/$CACHE_FILE$

# CodeStream plugin
# https://plugins.jetbrains.com/plugin/12206-codestream
.idea/codestream.xml

# Azure Toolkit for IntelliJ plugin
# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
.idea/**/azureSettings.xml

### Terraform ###
# Local .terraform directories
**/.terraform/*

# .tfstate files
*.tfstate
*.tfstate.*

# Crash log files
crash.log
crash.*.log

# Exclude all .tfvars files, which are likely to contain sensitive data, such as
# password, private keys, and other secrets. These should not be part of version
# control as they are data points which are potentially sensitive and subject
# to change depending on the environment.
*.tfvars
*.tfvars.json

# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json

# Include override files you do wish to add to version control using negated pattern
# !example_override.tf

# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*

# Ignore CLI configuration files
.terraformrc
terraform.rc

# End of https://www.toptal.com/developers/gitignore/api/go,terraform,intellij
2 changes: 1 addition & 1 deletion avro_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package avro_test

import "github.com/hamba/avro/v2"
import "github.com/justtrackio/avro/v2"

func ConfigTeardown() {
// Reset the caches
Expand Down
2 changes: 1 addition & 1 deletion bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"os"
"testing"

"github.com/hamba/avro/v2"
"github.com/justtrackio/avro/v2"
)

type Superhero struct {
Expand Down
176 changes: 173 additions & 3 deletions cmd/avrogen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package main

import (
"bytes"
"container/heap"
"encoding/json"
"errors"
"flag"
"fmt"
Expand All @@ -10,8 +12,8 @@ import (
"path/filepath"
"strings"

"github.com/hamba/avro/v2"
"github.com/hamba/avro/v2/gen"
"github.com/justtrackio/avro/v2"
"github.com/justtrackio/avro/v2/gen"
"golang.org/x/tools/imports"
)

Expand Down Expand Up @@ -89,7 +91,14 @@ func realMain(args []string, stdout, stderr io.Writer) int {
gen.WithFullSchema(cfg.FullSchema),
}
g := gen.NewGenerator(cfg.Pkg, tags, opts...)
for _, file := range flgs.Args() {

files, err := sortFiles(flgs.Args())
if err != nil {
_, _ = fmt.Fprintf(stderr, "Error: %v\n", err)
return 2
}

for _, file := range files {
schema, err := avro.ParseFiles(filepath.Clean(file))
if err != nil {
_, _ = fmt.Fprintf(stderr, "Error: %v\n", err)
Expand Down Expand Up @@ -118,6 +127,167 @@ func realMain(args []string, stdout, stderr io.Writer) int {
return 0
}

type doc struct {
Type avro.Type
Name string
Namespace string
Fields []field
}

type field struct {
Name string
Type any
}

type graph map[string]map[string]struct{}

func sortFiles(args []string) ([]string, error) {
deps := make(graph)

paths := make(map[string]string)
for _, filePath := range args {
file, err := os.Open(filepath.Clean(filePath))
if err != nil {
return nil, err
}

asBytes, err := io.ReadAll(file)
if err != nil {
return nil, err
}

var doc doc
if err := json.Unmarshal(asBytes, &doc); err != nil {
return nil, err
}

name := fmt.Sprintf("%s.%s", doc.Namespace, doc.Name)
paths[name] = filePath

if _, ok := deps[name]; !ok {
deps[name] = make(map[string]struct{})
}

for _, f := range doc.Fields {
for _, dep := range getDependencies(f.Type) {
if !strings.Contains(dep, ".") {
dep = fmt.Sprintf("%s.%s", doc.Namespace, dep)
}

if _, ok := deps[dep]; !ok {
deps[dep] = make(map[string]struct{})
}

deps[dep][name] = struct{}{}
}
}
}

indegree := make(map[string]int)
for node := range deps {
indegree[node] = 0
}

for _, neighbours := range deps {
for neighbor := range neighbours {
indegree[neighbor]++
}
}

sorted, err := kahnTopologicSort(deps, indegree)
if err != nil {
return nil, err
}

var sortedPaths []string
for _, it := range sorted {
if path, ok := paths[it]; ok {
sortedPaths = append(sortedPaths, path)
} else {
return nil, fmt.Errorf("could not find path for %s", it)
}
}

return sortedPaths, nil
}

type MinHeap []string

func (h MinHeap) Len() int { return len(h) }
func (h MinHeap) Less(i, j int) bool { return h[i] < h[j] }
func (h MinHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *MinHeap) Push(x any) {
*h = append(*h, x.(string))
}

func (h *MinHeap) Pop() any {
old := *h
n := len(old)
x := old[n-1]
*h = old[:n-1]
return x
}

func kahnTopologicSort(deps graph, indegree map[string]int) ([]string, error) {
queue := &MinHeap{}
heap.Init(queue)

var result []string

for node, deg := range indegree {
if deg == 0 {
heap.Push(queue, node)
}
}

for queue.Len() > 0 {
front := heap.Pop(queue)
node := front.(string)
result = append(result, node)

for neighbor := range deps[node] {
indegree[neighbor]--
if indegree[neighbor] == 0 {
heap.Push(queue, neighbor)
}
}
}

if len(result) != len(deps) {
return nil, fmt.Errorf("could not sort files, since they contain a cyclic dependency, or files are missing")
}

return result, nil
}

func isBuildIn(t string) bool {
switch avro.Type(t) {
case avro.String, avro.Bytes, avro.Fixed, avro.Int, avro.Long, avro.Double, avro.Boolean, avro.Null, avro.Float, avro.Array, avro.Map:
return true
}

return false
}

func getDependencies(t any) (deps []string) {
switch val := t.(type) {
case nil:
return deps
case string:
if !isBuildIn(val) {
deps = append(deps, val)
}
case []any:
for _, it := range val {
deps = append(deps, getDependencies(it)...)
}
case map[string]any:
deps = append(deps, getDependencies(val["type"])...)
}

return
}

func writeOut(filename string, stdout io.Writer, bytes []byte) error {
writer := stdout
if filename != "" {
Expand Down
Loading

0 comments on commit bd4efff

Please sign in to comment.