From 90cde6a34185c12e1ee08c38effb526f7167fa95 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Tue, 8 Aug 2023 21:32:09 +0200 Subject: [PATCH 01/18] Working on conv operator --- ops/opset13/conv.go | 104 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 ops/opset13/conv.go diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go new file mode 100644 index 0000000..d13cef9 --- /dev/null +++ b/ops/opset13/conv.go @@ -0,0 +1,104 @@ +package opset13 + +import ( + "fmt" + + "github.com/advancedclimatesystems/gonnx/onnx" + "github.com/advancedclimatesystems/gonnx/ops" + "gorgonia.org/tensor" +) + +type AutoPadSetting string + +const ( + NotSet AutoPadSetting = "NOTSET" + SameUpper AutoPadSetting = "SAME_UPPER" + SameLower AutoPadSetting = "SAME_LOWER" + Valid AutoPadSetting = "VALID" +) + +// Conv represents the ONNX conv operator. +type Conv struct { + // Type of padding to apply before doing the convolutions. + autoPad AutoPadSetting + + // Dilation value along each dimension of the filter. + dilations []int + + // Numer of groups the input channels and the output channels are divided into. + group int + + // Shape of the convolutional kernel. Can be present, but if not should be inferred (i.e. useless attribute). + kernelShape []int + + // Padding for the beginning and ending of each dimension. Cannot be used with autopad setting. + pads []int + + // Strides along each dimension. + strides []int +} + +// newConv creates a new conv operator. +func newConv() ops.Operator { + return &Conv{} +} + +// Init initializes the conv operator. +func (c *Conv) Init(attributes []*onnx.AttributeProto) error { + for _, attr := range attributes { + switch attr.GetName() { + case "auto_pad": + c.autoPad = AutoPadSetting(attr.GetS()) + case "linear_before_reset": + g.linearBeforeReset = ops.Int64ToBool(attr.GetI()) + default: + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) + } + } + + return nil +} + +// Apply applies the conv operator. +func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { + in1, in2, err := ops.MultidirectionalBroadcast(inputs[0], inputs[1]) + if err != nil { + return nil, err + } + + out, err := tensor.Conv(in1, in2) + if err != nil { + return nil, err + } + + return []tensor.Tensor{out}, nil +} + +// ValidateInputs validates the inputs that will be given to Apply for this operator. +func (c *Conv) ValidateInputs(inputs []tensor.Tensor) ([]tensor.Tensor, error) { + return ops.ValidateInputs(a, inputs) +} + +// GetMinInputs returns the minimum number of input tensors this operator expects. +func (c *Conv) GetMinInputs() int { + return 2 +} + +// GetMaxInputs returns the maximum number of input tensors this operator expects. +func (c *Conv) GetMaxInputs() int { + return 3 +} + +// GetInputTypeConstraints returns a list. Every element represents a set of allowed tensor dtypes +// for the corresponding input tensor. +func (c *Conv) GetInputTypeConstraints() [][]tensor.Dtype { + return [][]tensor.Dtype{ + {tensor.Uint32, tensor.Uint64, tensor.Int32, tensor.Int64, tensor.Float32, tensor.Float64}, + {tensor.Uint32, tensor.Uint64, tensor.Int32, tensor.Int64, tensor.Float32, tensor.Float64}, + } +} + +// String implements the stringer interface, and can be used to format errors or messages. +func (c *Conv) String() string { + return "conv operator" +} From 27b8b689a4419d16aa02f6e3ee2cb88d5cf8c183 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Wed, 9 Aug 2023 10:18:44 +0200 Subject: [PATCH 02/18] Added all attributes for conv operator --- ops/errors.go | 4 ++++ ops/opset13/conv.go | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/ops/errors.go b/ops/errors.go index ab25387..0135484 100644 --- a/ops/errors.go +++ b/ops/errors.go @@ -8,6 +8,10 @@ const UnknownAttributeErrTemplate = "%v: unknown attribute: %v" // an attribute that is not supported yet. const UnsupportedAttrErrTemplate = "%v: %v attribute not supported yet" +// InvalidAttrTemplate is used to format an error when a known attribute could not +// be parsed/interpreted correctly. +const InvalidAttrTemplate = "%v: attribute %v could not be parsed as %T" + // InvalidAttrCountErrTemplate is used to format an error when an operator // got the wrong amount of attributes. const InvalidAttrCountErrTemplate = "%v: expected %v attributes, got %d" diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index d13cef9..3b54a5d 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -49,8 +49,28 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { switch attr.GetName() { case "auto_pad": c.autoPad = AutoPadSetting(attr.GetS()) - case "linear_before_reset": - g.linearBeforeReset = ops.Int64ToBool(attr.GetI()) + case "dilations": + c.dilations, err := ops.AnyToIntSlice(attr.GetInts()) + if err != nil { + return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.dilations) + } + case "group": + c.group = attr.GetI() + case "kernel_shape": + c.kernelShape, err := ops.AnyToIntSlice(attr.GetInts()) + if err != nil { + return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.kernelShape) + } + case "pads": + c.pads, err := ops.AnyToIntSlice(attr.GetInts()) + if err != nil { + return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.pads) + } + case "strides": + c.strides, err := ops.AnyToIntSlice(attr.GetInts()) + if err != nil { + return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.strides) + } default: return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) } @@ -61,6 +81,14 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { // Apply applies the conv operator. func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { + X := inputs[0] + W := inputs[1] + + b := nil + if len(inputs) == 3 { + b = inputs[2] + } + in1, in2, err := ops.MultidirectionalBroadcast(inputs[0], inputs[1]) if err != nil { return nil, err From 3c729bc193c20a7750e64d61237513aff59f202c Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Mon, 28 Aug 2023 14:35:10 +0200 Subject: [PATCH 03/18] WIP on conv operator --- ops/opset13/conv.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index 3b54a5d..1d30e43 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -82,9 +82,9 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { // Apply applies the conv operator. func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { X := inputs[0] - W := inputs[1] + kernel := inputs[1] - b := nil + bias := nil if len(inputs) == 3 { b = inputs[2] } @@ -121,8 +121,9 @@ func (c *Conv) GetMaxInputs() int { // for the corresponding input tensor. func (c *Conv) GetInputTypeConstraints() [][]tensor.Dtype { return [][]tensor.Dtype{ - {tensor.Uint32, tensor.Uint64, tensor.Int32, tensor.Int64, tensor.Float32, tensor.Float64}, - {tensor.Uint32, tensor.Uint64, tensor.Int32, tensor.Int64, tensor.Float32, tensor.Float64}, + {tensor.Float32, tensor.Float64}, + {tensor.Float32, tensor.Float64}, + {tensor.Float32, tensor.Float64}, } } From 7c3e645397e599464637522249469771df6a2268 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Sun, 3 Sep 2023 20:53:11 +0200 Subject: [PATCH 04/18] Shit's hard --- ops/opset13/conv.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index 1d30e43..eeab898 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -49,6 +49,9 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { switch attr.GetName() { case "auto_pad": c.autoPad = AutoPadSetting(attr.GetS()) + if c.autoPad != "NOTSET" { + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) + } case "dilations": c.dilations, err := ops.AnyToIntSlice(attr.GetInts()) if err != nil { @@ -56,6 +59,9 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { } case "group": c.group = attr.GetI() + if c.group != 1 { + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) + } case "kernel_shape": c.kernelShape, err := ops.AnyToIntSlice(attr.GetInts()) if err != nil { @@ -72,7 +78,7 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.strides) } default: - return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) } } From 1de1b20580d57b30fb57b3e097b8bf9f8d9ab76f Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Tue, 10 Oct 2023 17:16:21 +0200 Subject: [PATCH 05/18] WIP on conv operator --- ops/opset13/conv.go | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index eeab898..e8d61ee 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -49,31 +49,31 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { switch attr.GetName() { case "auto_pad": c.autoPad = AutoPadSetting(attr.GetS()) - if c.autoPad != "NOTSET" { - return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) - } + if c.autoPad != "NOTSET" { + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) + } case "dilations": - c.dilations, err := ops.AnyToIntSlice(attr.GetInts()) + c.dilations, err = ops.AnyToIntSlice(attr.GetInts()) if err != nil { return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.dilations) } case "group": c.group = attr.GetI() - if c.group != 1 { - return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) - } + if c.group != 1 { + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) + } case "kernel_shape": - c.kernelShape, err := ops.AnyToIntSlice(attr.GetInts()) + c.kernelShape, err = ops.AnyToIntSlice(attr.GetInts()) if err != nil { return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.kernelShape) } case "pads": - c.pads, err := ops.AnyToIntSlice(attr.GetInts()) + c.pads, err = ops.AnyToIntSlice(attr.GetInts()) if err != nil { return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.pads) } case "strides": - c.strides, err := ops.AnyToIntSlice(attr.GetInts()) + c.strides, err = ops.AnyToIntSlice(attr.GetInts()) if err != nil { return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.strides) } @@ -90,22 +90,18 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { X := inputs[0] kernel := inputs[1] - bias := nil + var bias tensor.Tensor if len(inputs) == 3 { - b = inputs[2] - } - - in1, in2, err := ops.MultidirectionalBroadcast(inputs[0], inputs[1]) - if err != nil { - return nil, err + bias = inputs[2] } - out, err := tensor.Conv(in1, in2) - if err != nil { - return nil, err + // 2D Convolution where + if len(X.Shape()) == 4 { + } else { + return nil, fmt.Errorf("The convolution operator currently only supports 2D convolution, i.e. shape [N x C x H x W]") } - return []tensor.Tensor{out}, nil + return []tensor.Tensor{}, nil } // ValidateInputs validates the inputs that will be given to Apply for this operator. From 9e1bf0bddd25324acd5c42f786a5997e1981006f Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Sun, 15 Oct 2023 20:37:46 +0200 Subject: [PATCH 06/18] Set defaults for attributes --- ops/opset13/conv.go | 102 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 17 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index eeab898..2fb7bf5 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -49,9 +49,9 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { switch attr.GetName() { case "auto_pad": c.autoPad = AutoPadSetting(attr.GetS()) - if c.autoPad != "NOTSET" { - return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) - } + if c.autoPad != "NOTSET" { + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) + } case "dilations": c.dilations, err := ops.AnyToIntSlice(attr.GetInts()) if err != nil { @@ -59,9 +59,9 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { } case "group": c.group = attr.GetI() - if c.group != 1 { - return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) - } + if c.group != 1 { + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) + } case "kernel_shape": c.kernelShape, err := ops.AnyToIntSlice(attr.GetInts()) if err != nil { @@ -89,28 +89,32 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { X := inputs[0] kernel := inputs[1] - - bias := nil + var bias tensor.Tensor = nil if len(inputs) == 3 { - b = inputs[2] + bias = inputs[2] } - in1, in2, err := ops.MultidirectionalBroadcast(inputs[0], inputs[1]) - if err != nil { - return nil, err + if len(c.dilations) == 0 { + c.setDefaultDilations(X) } - - out, err := tensor.Conv(in1, in2) - if err != nil { - return nil, err + if len(c.kernelShape) == 0 { + c.setKernelShape(kernel) } + if len(c.pads) == 0 { + c.setDefaultPaddings(X) + } + if len(c.strides) == 0 { + c.setDefaultStrides(X) + } + + kernel = c.getDilatedKernel(kernel) return []tensor.Tensor{out}, nil } // ValidateInputs validates the inputs that will be given to Apply for this operator. func (c *Conv) ValidateInputs(inputs []tensor.Tensor) ([]tensor.Tensor, error) { - return ops.ValidateInputs(a, inputs) + return ops.ValidateInputs(c, inputs) } // GetMinInputs returns the minimum number of input tensors this operator expects. @@ -137,3 +141,67 @@ func (c *Conv) GetInputTypeConstraints() [][]tensor.Dtype { func (c *Conv) String() string { return "conv operator" } + +// setDefaultDilations sets the dilations attribute to the default. Can be called when no +// dilations were set when initializing. +func (c *Conv) setDefaultDilations(X tensor.Tensor) { + nDims := len(X.Shape()[2:]) + + dilations := make([]int, nDims) + for i := 0; i < nDims; i++ { + dilations[i] = 1 + } + + c.dilations = dilations +} + +// setKernelShape infers the shape of the kernel when it was not given in the attributes. +func (c *Conv) setKernelShape(kernel tensor.Tensor) { + c.kernelShape = kernel.Shape()[2:] +} + +// setDefaultPaddings sets default paddings as attribute. Can be called when no paddings +// were set during initialization. +func (c *Conv) setDefaultPaddings(X tensor.Tensor) { + paddingLength := len(X.Shape()[2:]) * 2 + + pads := make([]int, paddingLength) + for i := 0; i < paddingLength; i++ { + pads[i] = 0 + } + + c.pads = pads +} + +// setDefaultStrides sets default strides as attribute. Can be called when no strides +// were set during initialization. +func (c *Conv) setDefaultStrides(X tensor.Tensor) { + nDims := len(X.Shape()[2:]) + + strides := make([]int, nDims) + for i := 0; i < nDims; i++ { + strides[i] = 1 + } + + c.strides = strides +} + +// getDilatedKernel creates a new kernel given the `dilations` attribute of this +// conv operator. A dilated kernel basically means inserting zeros in between +// the kernels, i.e. a 2D kernel like: +// +// 1 2 +// 3 4 +// +// Dilated by one in both dimensions yields a new kernel of: +// +// 1 0 2 +// 0 0 0 +// 3 0 4 +// +// This function updates the given kernel and dilates it by the given amount +// for each dimensions separately. It returns a new tensor with the new kernel. +func (c *Conv) getDilatedKernel(kernel tensor.Tensor) tensor.Tensor { + // TODO + return kernel +} From c8f76b8650a55592ab061bed3c9ae4971a1c1999 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Mon, 16 Oct 2023 11:06:49 +0200 Subject: [PATCH 07/18] Finished computation of dilated kernel --- ops/opset13/conv.go | 66 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index ca67770..4e0d837 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -45,12 +45,13 @@ func newConv() ops.Operator { // Init initializes the conv operator. func (c *Conv) Init(attributes []*onnx.AttributeProto) error { + var err error for _, attr := range attributes { switch attr.GetName() { case "auto_pad": c.autoPad = AutoPadSetting(attr.GetS()) if c.autoPad != "NOTSET" { - return fmt.Errorf(ops.UnsupportedAttrErrTemplate, g, attr.GetName()) + return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) } case "dilations": c.dilations, err = ops.AnyToIntSlice(attr.GetInts()) @@ -58,7 +59,7 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { return fmt.Errorf(ops.InvalidAttrTemplate, c, attr.GetName(), c.dilations) } case "group": - c.group = attr.GetI() + c.group = int(attr.GetI()) if c.group != 1 { return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) } @@ -107,7 +108,10 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { c.setDefaultStrides(X) } - kernel = c.getDilatedKernel(kernel) + kernel, err := c.getDilatedKernel(kernel) + if err != nil { + return nil, err + } // 2D Convolution where if len(X.Shape()) == 4 { @@ -207,7 +211,57 @@ func (c *Conv) setDefaultStrides(X tensor.Tensor) { // // This function updates the given kernel and dilates it by the given amount // for each dimensions separately. It returns a new tensor with the new kernel. -func (c *Conv) getDilatedKernel(kernel tensor.Tensor) tensor.Tensor { - // TODO - return kernel +func (c *Conv) getDilatedKernel(kernel tensor.Tensor) (tensor.Tensor, error) { + oldKernelShape := kernel.Shape() + newKernelShape := make([]int, len(oldKernelShape)) + + // Add the non spatial dimensions of the kernel, i.e. the number of + // kernels (index 0) and the number of channels (index 1). These + // dimensions do not have to be dilated. + nNonSpatialDims := 2 + for i := 0; i < nNonSpatialDims; i++ { + newKernelShape[i] = oldKernelShape[i] + } + + // Add the dilated spatial dimensions of the kernel, i.e. in the case + // of 2D images these are the width and height dimensions. + for i, dilation := range c.dilations { + oldKernelDim := oldKernelShape[nNonSpatialDims+i] + newKernelShape[nNonSpatialDims+i] = oldKernelDim + (oldKernelDim-1)*(dilation-1) + } + + newKernel := tensor.NewDense(kernel.Dtype(), newKernelShape) + newKernel.Zero() + + // Now we fill the empty kernel with the original kernel values at the + // right positions. + iterator := kernel.Iterator() + for iterator.Reset(); !iterator.Done(); iterator.Next() { + oldCoords := iterator.Coord() + value, err := kernel.At(oldCoords...) + if err != nil { + return nil, err + } + + newCoords := c.getNewKernelCoords(oldCoords, kernel.Shape(), newKernel.Shape()) + newKernel.SetAt(value, newCoords...) + } + + c.setKernelShape(newKernel) + return newKernel, nil +} + +func (c *Conv) getNewKernelCoords(oldCoords, oldShape, newShape []int) []int { + newCoords := make([]int, len(oldCoords)) + + nNonSpatialDims := 2 + for i := 0; i < nNonSpatialDims; i++ { + newCoords[i] = oldCoords[i] + } + + for i, dilation := range c.dilations { + newCoords[nNonSpatialDims+i] = oldCoords[nNonSpatialDims+i] * dilation + } + + return newCoords } From 8cd59d2c2a9939c39c22c00249ca361d68197485 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Mon, 16 Oct 2023 17:37:58 +0200 Subject: [PATCH 08/18] Start of 1D conv implementation --- ops/opset13/conv.go | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index 4e0d837..ec3e3b1 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -114,9 +114,11 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { } // 2D Convolution where - if len(X.Shape()) == 4 { + if len(X.Shape()) == 3 { + c.applyConv1D(X, kernel, bias) + } else if len(X.Shape()) == 4 { } else { - return nil, fmt.Errorf("The convolution operator currently only supports 2D convolution, i.e. shape [N x C x H x W]") + return nil, fmt.Errorf("The convolution operator currently only supports 1D or 2D convolution, i.e. shape [N x C x H (x W)]") } return []tensor.Tensor{}, nil @@ -243,7 +245,7 @@ func (c *Conv) getDilatedKernel(kernel tensor.Tensor) (tensor.Tensor, error) { return nil, err } - newCoords := c.getNewKernelCoords(oldCoords, kernel.Shape(), newKernel.Shape()) + newCoords := c.getNewCoordsAfterDilation(oldCoords, kernel.Shape()) newKernel.SetAt(value, newCoords...) } @@ -251,7 +253,10 @@ func (c *Conv) getDilatedKernel(kernel tensor.Tensor) (tensor.Tensor, error) { return newKernel, nil } -func (c *Conv) getNewKernelCoords(oldCoords, oldShape, newShape []int) []int { +// getNewCoordsAfterDilation returns the new coordinates of a value given the old coordinates of that +// value in the old kernel and its shape. The new coordinates can be used to store the value/weight +// in the dilated kernel. +func (c *Conv) getNewCoordsAfterDilation(oldCoords, oldShape []int) []int { newCoords := make([]int, len(oldCoords)) nNonSpatialDims := 2 @@ -265,3 +270,33 @@ func (c *Conv) getNewKernelCoords(oldCoords, oldShape, newShape []int) []int { return newCoords } + +// Applies 1D convolution to tensor X with the 'kernel' tensor. +// X will have 3 dimensions: [N, C, H] where N is the batch size, C is the number +// of channels and H is the number of dimensions on which to apply the convolutions. +// The kernel will have shape [kernelDim], where 'kernelDim' is the size of the kernel +// size of the kernel. +func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) { + dimH := x.Shape()[2] + kernelSize := c.kernelShape[0] + strideSize := c.strides[0] + + outputDim := ((dimH - kernelSize + c.pads[0] + c.pads[1]) / strideSize) + 1 + outputShape := []int{x.Shape()[0], kernel.Shape()[0], outputDim} + out := tensor.Tensor(tensor.New(tensor.WithShape(outputShape...))) + out.Zero() + + if bias != nil { + err := bias.Reshape(1, bias.Shape()[0], 1) + if err != nil { + return nil, err + } + + out, err = tensor.Add(out, bias) + if err != nil { + return nil, err + } + } + + return out, nil +} From 14ed77c2e52083da6cb9bbccc7b1eb8fbe3bd3af Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Tue, 17 Oct 2023 09:21:47 +0200 Subject: [PATCH 09/18] Almost finished 1D convolution --- ops/opset13/conv.go | 85 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index ec3e3b1..6290cfe 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -88,7 +88,7 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { // Apply applies the conv operator. func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { - X := inputs[0] + x := inputs[0] kernel := inputs[1] var bias tensor.Tensor = nil if len(inputs) == 3 { @@ -96,16 +96,16 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { } if len(c.dilations) == 0 { - c.setDefaultDilations(X) + c.setDefaultDilations(x) } if len(c.kernelShape) == 0 { c.setKernelShape(kernel) } if len(c.pads) == 0 { - c.setDefaultPaddings(X) + c.setDefaultPaddings(x) } if len(c.strides) == 0 { - c.setDefaultStrides(X) + c.setDefaultStrides(x) } kernel, err := c.getDilatedKernel(kernel) @@ -113,15 +113,15 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { return nil, err } - // 2D Convolution where - if len(X.Shape()) == 3 { - c.applyConv1D(X, kernel, bias) - } else if len(X.Shape()) == 4 { + var out tensor.Tensor + if len(x.Shape()) == 3 { + out, err = c.applyConv1D(x, kernel, bias) + } else if len(x.Shape()) == 4 { } else { return nil, fmt.Errorf("The convolution operator currently only supports 1D or 2D convolution, i.e. shape [N x C x H (x W)]") } - return []tensor.Tensor{}, nil + return []tensor.Tensor{out}, nil } // ValidateInputs validates the inputs that will be given to Apply for this operator. @@ -283,7 +283,7 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) outputDim := ((dimH - kernelSize + c.pads[0] + c.pads[1]) / strideSize) + 1 outputShape := []int{x.Shape()[0], kernel.Shape()[0], outputDim} - out := tensor.Tensor(tensor.New(tensor.WithShape(outputShape...))) + out := tensor.Tensor(tensor.NewDense(x.Dtype(), outputShape)) out.Zero() if bias != nil { @@ -292,11 +292,76 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) return nil, err } + out, bias, err := ops.MultidirectionalBroadcast(out, bias) + if err != nil { + return nil, err + } + out, err = tensor.Add(out, bias) if err != nil { return nil, err } } + paddedX, err := c.padInput(x) + if err != nil { + return nil, err + } + + nBatches := x.Shape()[0] + nChannels := x.Shape()[1] + nKernels := kernel.Shape()[0] + for batchIdx := 0; batchIdx < nBatches; batchIdx++ { + for kernelIdx := 0; kernelIdx < nKernels; kernelIdx++ { + for channelIdx := 0; channelIdx < nChannels; channelIdx++ { + subKernel, err := getSubKernel(kernel, kernelIdx, channelIdx) + if err != nil { + return nil, err + } + + // TODO extract image patch from paddedX + // TODO multiply kernel with image patch + // TODO update values in output tensor + + val, err := out.At(batchIdx, kernelIdx, outputIdx) + } + } + } + return out, nil } + +func (c *Conv) padInput(x tensor.Tensor) (tensor.Tensor, error) { + nSpatialDims := len(x.Shape()[2:]) + nNonSpatialDims := 2 + + var err error + for i := 0; i < nSpatialDims; i++ { + padsBeforeShape := x.Shape().Clone() + padsBeforeShape[nNonSpatialDims+i] = c.pads[i] + zerosBefore := tensor.Tensor(tensor.NewDense(x.Dtype(), padsBeforeShape)) + zerosBefore.Zero() + + padsAfterShape := x.Shape().Clone() + padsAfterShape[nNonSpatialDims+i] = c.pads[i+nSpatialDims] + zerosAfter := tensor.Tensor(tensor.NewDense(x.Dtype(), padsAfterShape)) + zerosAfter.Zero() + + x, err = tensor.Concat(nNonSpatialDims+i, zerosBefore, x, zerosAfter) + if err != nil { + return nil, err + } + } + + return x, nil +} + +// getSubKernel returns a sub kernel of the given kernel. The main kernel is assumed to have +// shape [M, C, H, ...] where M is the number of kernels, C is the number of channels, and the +// rest are spatial dimensions. This method returns a subkernel of shape [1, 1, H, ...]. +func getSubKernel(kernel tensor.Tensor, kernelIdx, channelIdx int) (tensor.Tensor, error) { + return kernel.Slice( + ops.NewSlicer(kernelIdx, kernelIdx+1), + ops.NewSlicer(channelIdx, channelIdx+1), + ) +} From ea5f693b54367338aba715a5381c0ff6d0f2a386 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Thu, 19 Oct 2023 11:33:31 +0200 Subject: [PATCH 10/18] Working 2D conv! --- ops/opset13/conv.go | 241 +++++++++++++++++++++++++++++++----- ops/opset13/opset13.go | 3 + ops/opset13/opset13_test.go | 5 + ops_test.go | 4 + 4 files changed, 220 insertions(+), 33 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index 6290cfe..bbf7201 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -40,19 +40,19 @@ type Conv struct { // newConv creates a new conv operator. func newConv() ops.Operator { - return &Conv{} + return &Conv{ + autoPad: "NOTSET", + } } // Init initializes the conv operator. func (c *Conv) Init(attributes []*onnx.AttributeProto) error { var err error + for _, attr := range attributes { switch attr.GetName() { case "auto_pad": c.autoPad = AutoPadSetting(attr.GetS()) - if c.autoPad != "NOTSET" { - return fmt.Errorf(ops.UnsupportedAttrErrTemplate, c, attr.GetName()) - } case "dilations": c.dilations, err = ops.AnyToIntSlice(attr.GetInts()) if err != nil { @@ -90,7 +90,8 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { x := inputs[0] kernel := inputs[1] - var bias tensor.Tensor = nil + var bias tensor.Tensor + if len(inputs) == 3 { bias = inputs[2] } @@ -98,12 +99,15 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { if len(c.dilations) == 0 { c.setDefaultDilations(x) } + if len(c.kernelShape) == 0 { c.setKernelShape(kernel) } + if len(c.pads) == 0 { c.setDefaultPaddings(x) } + if len(c.strides) == 0 { c.setDefaultStrides(x) } @@ -113,14 +117,23 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { return nil, err } + if c.autoPad != "NOTSET" { + c.setPaddingWithAutoPad(x) + } + var out tensor.Tensor if len(x.Shape()) == 3 { out, err = c.applyConv1D(x, kernel, bias) } else if len(x.Shape()) == 4 { + out, err = c.applyConv2D(x, kernel, bias) } else { return nil, fmt.Errorf("The convolution operator currently only supports 1D or 2D convolution, i.e. shape [N x C x H (x W)]") } + if err != nil { + return nil, err + } + return []tensor.Tensor{out}, nil } @@ -198,6 +211,36 @@ func (c *Conv) setDefaultStrides(X tensor.Tensor) { c.strides = strides } +func (c *Conv) setPaddingWithAutoPad(x tensor.Tensor) { + if c.autoPad == "NOTSET" { + return + } + + inputShape := x.Shape() + nDims := len(inputShape) + nNonSpatialDims := 2 + nSpatialDims := nDims - nNonSpatialDims + + c.pads = make([]int, nSpatialDims*2) + + for i := 0; i < nSpatialDims; i++ { + dim := inputShape[i] + targetSize := (dim + c.strides[i] - 1) / c.strides[i] + padNeeded := (targetSize-1)*c.strides[i] + c.kernelShape[i] - dim + + var padHead int + if c.autoPad == "SAME_LOWER" { + padHead = (padNeeded + 1) / 2 + } else { + padHead = padNeeded / 2 + } + + padTail := padNeeded - padHead + c.pads[i] = padHead + c.pads[i+nSpatialDims] = padTail + } +} + // getDilatedKernel creates a new kernel given the `dilations` attribute of this // conv operator. A dilated kernel basically means inserting zeros in between // the kernels, i.e. a 2D kernel like: @@ -292,7 +335,7 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) return nil, err } - out, bias, err := ops.MultidirectionalBroadcast(out, bias) + out, bias, err = ops.MultidirectionalBroadcast(out, bias) if err != nil { return nil, err } @@ -309,59 +352,191 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) } nBatches := x.Shape()[0] - nChannels := x.Shape()[1] nKernels := kernel.Shape()[0] + for batchIdx := 0; batchIdx < nBatches; batchIdx++ { for kernelIdx := 0; kernelIdx < nKernels; kernelIdx++ { - for channelIdx := 0; channelIdx < nChannels; channelIdx++ { - subKernel, err := getSubKernel(kernel, kernelIdx, channelIdx) + subKernel, err := kernel.Slice(ops.NewSlicer(kernelIdx, kernelIdx+1)) + if err != nil { + return nil, err + } + + for i := 0; i < paddedX.Shape()[2]; i += strideSize { + subImage, err := c.getSubImage(paddedX, batchIdx, i) + if err != nil { + return nil, err + } + + convResult, err := tensor.Mul(subImage, subKernel) if err != nil { return nil, err } - // TODO extract image patch from paddedX - // TODO multiply kernel with image patch - // TODO update values in output tensor + convValue, err := tensor.Sum(convResult) + if err != nil { + return nil, err + } + + dimOutputIdx := i / strideSize + + err = out.SetAt(convValue.ScalarValue(), batchIdx, kernelIdx, dimOutputIdx) + if err != nil { + return nil, err + } + } + } + } + + return out, nil +} + +// Applies 2D convolution to tensor X with the 'kernel' tensor. +// X will have 4 dimensions: [N, C, H, W] where N is the batch size, C is the number +// of channels, H and W are the height and width dimensions on which to apply the convolutions. +// The kernel will have shape [M, C, H, W]. +func (c *Conv) applyConv2D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) { + dimH := x.Shape()[2] + dimW := x.Shape()[3] + + kernelHSize := c.kernelShape[0] + kernelWSize := c.kernelShape[1] + strideHSize := c.strides[0] + strideWSize := c.strides[1] + + outputHDim := ((dimH - kernelHSize + c.pads[0] + c.pads[2]) / strideHSize) + 1 + outputWDim := ((dimW - kernelWSize + c.pads[1] + c.pads[3]) / strideWSize) + 1 + outputShape := []int{x.Shape()[0], kernel.Shape()[0], outputHDim, outputWDim} + out := tensor.Tensor(tensor.NewDense(x.Dtype(), outputShape)) + out.Zero() + + paddedX, err := c.padInput(x) + if err != nil { + return nil, err + } + + nBatches := x.Shape()[0] + nKernels := kernel.Shape()[0] + + for batchIdx := 0; batchIdx < nBatches; batchIdx++ { + for kernelIdx := 0; kernelIdx < nKernels; kernelIdx++ { + subKernel, err := kernel.Slice(ops.NewSlicer(kernelIdx, kernelIdx+1)) + if err != nil { + return nil, err + } + + for h := 0; h < paddedX.Shape()[2]; h += strideHSize { + dimHOutputIdx := h / strideHSize + if dimHOutputIdx >= outputHDim { + continue + } - val, err := out.At(batchIdx, kernelIdx, outputIdx) + for w := 0; w < paddedX.Shape()[2]; w += strideWSize { + dimWOutputIdx := w / strideWSize + if dimWOutputIdx >= outputWDim { + continue + } + + subImage, err := c.getSubImage(paddedX, batchIdx, h, w) + if err != nil { + return nil, err + } + + copiedSubKernel := subKernel.Materialize() + copiedSubImage := subImage.Materialize() + + convResult, err := tensor.Mul(copiedSubImage, copiedSubKernel) + if err != nil { + return nil, err + } + + convValue, err := tensor.Sum(convResult) + if err != nil { + return nil, err + } + + err = out.SetAt(convValue.ScalarValue(), batchIdx, kernelIdx, dimHOutputIdx, dimWOutputIdx) + if err != nil { + return nil, err + } + } } } } + if bias != nil { + err := bias.Reshape(1, bias.Shape()[0], 1, 1) + if err != nil { + return nil, err + } + + out, bias, err = ops.MultidirectionalBroadcast(out, bias) + if err != nil { + return nil, err + } + + out, err = tensor.Add(out, bias) + if err != nil { + return nil, err + } + } + return out, nil } func (c *Conv) padInput(x tensor.Tensor) (tensor.Tensor, error) { + var err error + nSpatialDims := len(x.Shape()[2:]) nNonSpatialDims := 2 - var err error for i := 0; i < nSpatialDims; i++ { - padsBeforeShape := x.Shape().Clone() - padsBeforeShape[nNonSpatialDims+i] = c.pads[i] - zerosBefore := tensor.Tensor(tensor.NewDense(x.Dtype(), padsBeforeShape)) - zerosBefore.Zero() + if c.pads[i] != 0 { + padsBeforeShape := x.Shape().Clone() + padsBeforeShape[nNonSpatialDims+i] = c.pads[i] + zerosBefore := tensor.Tensor(tensor.NewDense(x.Dtype(), padsBeforeShape)) + zerosBefore.Zero() - padsAfterShape := x.Shape().Clone() - padsAfterShape[nNonSpatialDims+i] = c.pads[i+nSpatialDims] - zerosAfter := tensor.Tensor(tensor.NewDense(x.Dtype(), padsAfterShape)) - zerosAfter.Zero() + x, err = tensor.Concat(nNonSpatialDims+i, zerosBefore, x) + if err != nil { + return nil, err + } + } - x, err = tensor.Concat(nNonSpatialDims+i, zerosBefore, x, zerosAfter) - if err != nil { - return nil, err + if c.pads[i+nSpatialDims] != 0 { + padsAfterShape := x.Shape().Clone() + padsAfterShape[nNonSpatialDims+i] = c.pads[i+nSpatialDims] + zerosAfter := tensor.Tensor(tensor.NewDense(x.Dtype(), padsAfterShape)) + zerosAfter.Zero() + + x, err = tensor.Concat(nNonSpatialDims+i, x, zerosAfter) + if err != nil { + return nil, err + } } + } return x, nil } -// getSubKernel returns a sub kernel of the given kernel. The main kernel is assumed to have -// shape [M, C, H, ...] where M is the number of kernels, C is the number of channels, and the -// rest are spatial dimensions. This method returns a subkernel of shape [1, 1, H, ...]. -func getSubKernel(kernel tensor.Tensor, kernelIdx, channelIdx int) (tensor.Tensor, error) { - return kernel.Slice( - ops.NewSlicer(kernelIdx, kernelIdx+1), - ops.NewSlicer(channelIdx, channelIdx+1), - ) +// getSubImage returns a the subimage for a specific example in the batch, based on the +// kernel shape and the given start coordinates. The resulting sub image will be of +// shape [1, C, kernelShape[0], kernelShape[1], ...]. +func (c *Conv) getSubImage(x tensor.Tensor, batchIdx int, startSpatialCoords ...int) (tensor.View, error) { + if len(startSpatialCoords) != len(c.kernelShape) { + return nil, fmt.Errorf("expected the coordinates to have the same number of dimensions as the kernel") + } + + slices := []tensor.Slice{ + ops.NewSlicer(batchIdx, batchIdx+1), + nil, // Take all channels at once. + } + + for i := 0; i < len(c.kernelShape); i++ { + dimStartIdx := startSpatialCoords[i] + dimKernelSize := c.kernelShape[i] + slices = append(slices, ops.NewSlicer(dimStartIdx, dimStartIdx+dimKernelSize)) + } + + return x.Slice(slices...) } diff --git a/ops/opset13/opset13.go b/ops/opset13/opset13.go index 7bbb74c..32cfeb2 100644 --- a/ops/opset13/opset13.go +++ b/ops/opset13/opset13.go @@ -13,6 +13,7 @@ var operators13 = map[string]func() ops.Operator{ "Concat": newConcat, "Constant": newConstant, "ConstantOfShape": newConstantOfShape, + "Conv": newConv, "Div": newDiv, "Gather": newGather, "Gemm": newGemm, @@ -38,6 +39,7 @@ func GetOperator(opType string) (ops.Operator, error) { if opInit, ok := operators13[opType]; ok { return opInit(), nil } + return nil, fmt.Errorf(ops.UnknowOpTypeErrTemplate, opType) } @@ -47,5 +49,6 @@ func GetOpNames() []string { for opName := range operators13 { opList = append(opList, opName) } + return opList } diff --git a/ops/opset13/opset13_test.go b/ops/opset13/opset13_test.go index c826e46..21c51d7 100644 --- a/ops/opset13/opset13_test.go +++ b/ops/opset13/opset13_test.go @@ -44,6 +44,11 @@ func TestGetOperator(t *testing.T) { newConstantOfShape(), nil, }, + { + "Conv", + newConv(), + nil, + }, { "Div", newDiv(), diff --git a/ops_test.go b/ops_test.go index 6fb94fd..adc703a 100644 --- a/ops_test.go +++ b/ops_test.go @@ -274,6 +274,10 @@ var expectedTests = []string{ "test_constant", "test_constantofshape_float_ones", "test_constantofshape_int_zeros", + "test_conv_with_autopad_same", + "test_conv_with_strides_and_asymmetric_padding", + "test_conv_with_strides_no_padding", + "test_conv_with_strides_padding", "test_div", "test_div_bcast", "test_div_example", From dac0dc40f866e6365b419aca1af3cc0a0857dcb5 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Fri, 20 Oct 2023 18:24:11 +0200 Subject: [PATCH 11/18] Working convolution operator --- .gitattributes | 1 + .gitignore | 2 + ops/opset13/conv.go | 203 ++++++++++-------- ops_test.go | 8 +- .../onnx_models/mnist-8-opset13.onnx | 3 + sample_models/requirements.txt | 6 +- 6 files changed, 133 insertions(+), 90 deletions(-) create mode 100644 .gitattributes create mode 100644 sample_models/onnx_models/mnist-8-opset13.onnx diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3fe60cc --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +sample_models/onnx_models/mnist-8-opset13.onnx filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 37ffce7..fa81ed7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ test_data/ .coverage.out + +sample_models/.env diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index bbf7201..eb1bebf 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -17,6 +17,8 @@ const ( Valid AutoPadSetting = "VALID" ) +const nNonSpatialDims = 2 + // Conv represents the ONNX conv operator. type Conv struct { // Type of padding to apply before doing the convolutions. @@ -90,9 +92,9 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { x := inputs[0] kernel := inputs[1] - var bias tensor.Tensor - if len(inputs) == 3 { + var bias tensor.Tensor + if len(inputs) == c.GetMaxInputs() { bias = inputs[2] } @@ -122,11 +124,13 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { } var out tensor.Tensor - if len(x.Shape()) == 3 { - out, err = c.applyConv1D(x, kernel, bias) - } else if len(x.Shape()) == 4 { - out, err = c.applyConv2D(x, kernel, bias) - } else { + + switch len(x.Shape()) { + case 3: + out, err = c.applyConv1D(x, kernel) + case 4: + out, err = c.applyConv2D(x, kernel) + default: return nil, fmt.Errorf("The convolution operator currently only supports 1D or 2D convolution, i.e. shape [N x C x H (x W)]") } @@ -134,6 +138,13 @@ func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { return nil, err } + if bias != nil { + out, err = c.addBias(out, bias) + if err != nil { + return nil, err + } + } + return []tensor.Tensor{out}, nil } @@ -169,8 +180,8 @@ func (c *Conv) String() string { // setDefaultDilations sets the dilations attribute to the default. Can be called when no // dilations were set when initializing. -func (c *Conv) setDefaultDilations(X tensor.Tensor) { - nDims := len(X.Shape()[2:]) +func (c *Conv) setDefaultDilations(x tensor.Tensor) { + nDims := len(x.Shape()[2:]) dilations := make([]int, nDims) for i := 0; i < nDims; i++ { @@ -187,8 +198,8 @@ func (c *Conv) setKernelShape(kernel tensor.Tensor) { // setDefaultPaddings sets default paddings as attribute. Can be called when no paddings // were set during initialization. -func (c *Conv) setDefaultPaddings(X tensor.Tensor) { - paddingLength := len(X.Shape()[2:]) * 2 +func (c *Conv) setDefaultPaddings(x tensor.Tensor) { + paddingLength := len(x.Shape()[2:]) * 2 pads := make([]int, paddingLength) for i := 0; i < paddingLength; i++ { @@ -200,8 +211,8 @@ func (c *Conv) setDefaultPaddings(X tensor.Tensor) { // setDefaultStrides sets default strides as attribute. Can be called when no strides // were set during initialization. -func (c *Conv) setDefaultStrides(X tensor.Tensor) { - nDims := len(X.Shape()[2:]) +func (c *Conv) setDefaultStrides(x tensor.Tensor) { + nDims := len(x.Shape()[2:]) strides := make([]int, nDims) for i := 0; i < nDims; i++ { @@ -211,6 +222,8 @@ func (c *Conv) setDefaultStrides(X tensor.Tensor) { c.strides = strides } +// setPaddingWithAutoPad sets the padding attribute of the operator based on +// the input tensor `x`, the shape of the kernel and the strides. func (c *Conv) setPaddingWithAutoPad(x tensor.Tensor) { if c.autoPad == "NOTSET" { return @@ -218,7 +231,6 @@ func (c *Conv) setPaddingWithAutoPad(x tensor.Tensor) { inputShape := x.Shape() nDims := len(inputShape) - nNonSpatialDims := 2 nSpatialDims := nDims - nNonSpatialDims c.pads = make([]int, nSpatialDims*2) @@ -263,7 +275,6 @@ func (c *Conv) getDilatedKernel(kernel tensor.Tensor) (tensor.Tensor, error) { // Add the non spatial dimensions of the kernel, i.e. the number of // kernels (index 0) and the number of channels (index 1). These // dimensions do not have to be dilated. - nNonSpatialDims := 2 for i := 0; i < nNonSpatialDims; i++ { newKernelShape[i] = oldKernelShape[i] } @@ -281,28 +292,40 @@ func (c *Conv) getDilatedKernel(kernel tensor.Tensor) (tensor.Tensor, error) { // Now we fill the empty kernel with the original kernel values at the // right positions. iterator := kernel.Iterator() - for iterator.Reset(); !iterator.Done(); iterator.Next() { + iterator.Reset() + + for !iterator.Done() { oldCoords := iterator.Coord() + value, err := kernel.At(oldCoords...) if err != nil { return nil, err } - newCoords := c.getNewCoordsAfterDilation(oldCoords, kernel.Shape()) - newKernel.SetAt(value, newCoords...) + newCoords := c.getNewCoordsAfterDilation(oldCoords) + + err = newKernel.SetAt(value, newCoords...) + if err != nil { + return nil, err + } + + _, err = iterator.Next() + if err != nil { + return nil, err + } } c.setKernelShape(newKernel) + return newKernel, nil } // getNewCoordsAfterDilation returns the new coordinates of a value given the old coordinates of that // value in the old kernel and its shape. The new coordinates can be used to store the value/weight // in the dilated kernel. -func (c *Conv) getNewCoordsAfterDilation(oldCoords, oldShape []int) []int { +func (c *Conv) getNewCoordsAfterDilation(oldCoords []int) []int { newCoords := make([]int, len(oldCoords)) - nNonSpatialDims := 2 for i := 0; i < nNonSpatialDims; i++ { newCoords[i] = oldCoords[i] } @@ -319,33 +342,11 @@ func (c *Conv) getNewCoordsAfterDilation(oldCoords, oldShape []int) []int { // of channels and H is the number of dimensions on which to apply the convolutions. // The kernel will have shape [kernelDim], where 'kernelDim' is the size of the kernel // size of the kernel. -func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) { - dimH := x.Shape()[2] - kernelSize := c.kernelShape[0] - strideSize := c.strides[0] - - outputDim := ((dimH - kernelSize + c.pads[0] + c.pads[1]) / strideSize) + 1 - outputShape := []int{x.Shape()[0], kernel.Shape()[0], outputDim} +func (c *Conv) applyConv1D(x, kernel tensor.Tensor) (tensor.Tensor, error) { + outputShape := c.getOutputShape(x, kernel) out := tensor.Tensor(tensor.NewDense(x.Dtype(), outputShape)) out.Zero() - if bias != nil { - err := bias.Reshape(1, bias.Shape()[0], 1) - if err != nil { - return nil, err - } - - out, bias, err = ops.MultidirectionalBroadcast(out, bias) - if err != nil { - return nil, err - } - - out, err = tensor.Add(out, bias) - if err != nil { - return nil, err - } - } - paddedX, err := c.padInput(x) if err != nil { return nil, err @@ -353,6 +354,8 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) nBatches := x.Shape()[0] nKernels := kernel.Shape()[0] + strideSize := c.strides[0] + outputHDim := outputShape[nNonSpatialDims] for batchIdx := 0; batchIdx < nBatches; batchIdx++ { for kernelIdx := 0; kernelIdx < nKernels; kernelIdx++ { @@ -361,8 +364,13 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) return nil, err } - for i := 0; i < paddedX.Shape()[2]; i += strideSize { - subImage, err := c.getSubImage(paddedX, batchIdx, i) + for h := 0; h < paddedX.Shape()[2]; h += strideSize { + dimHOutputIdx := h / strideSize + if dimHOutputIdx > outputHDim { + continue + } + + subImage, err := c.getSubImage(paddedX, batchIdx, h) if err != nil { return nil, err } @@ -377,9 +385,7 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) return nil, err } - dimOutputIdx := i / strideSize - - err = out.SetAt(convValue.ScalarValue(), batchIdx, kernelIdx, dimOutputIdx) + err = out.SetAt(convValue.ScalarValue(), batchIdx, kernelIdx, dimHOutputIdx) if err != nil { return nil, err } @@ -394,21 +400,14 @@ func (c *Conv) applyConv1D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) // X will have 4 dimensions: [N, C, H, W] where N is the batch size, C is the number // of channels, H and W are the height and width dimensions on which to apply the convolutions. // The kernel will have shape [M, C, H, W]. -func (c *Conv) applyConv2D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) { - dimH := x.Shape()[2] - dimW := x.Shape()[3] - - kernelHSize := c.kernelShape[0] - kernelWSize := c.kernelShape[1] - strideHSize := c.strides[0] - strideWSize := c.strides[1] - - outputHDim := ((dimH - kernelHSize + c.pads[0] + c.pads[2]) / strideHSize) + 1 - outputWDim := ((dimW - kernelWSize + c.pads[1] + c.pads[3]) / strideWSize) + 1 - outputShape := []int{x.Shape()[0], kernel.Shape()[0], outputHDim, outputWDim} +func (c *Conv) applyConv2D(x, kernel tensor.Tensor) (tensor.Tensor, error) { + outputShape := c.getOutputShape(x, kernel) out := tensor.Tensor(tensor.NewDense(x.Dtype(), outputShape)) out.Zero() + outputHDim := outputShape[nNonSpatialDims] + outputWDim := outputShape[nNonSpatialDims+1] + paddedX, err := c.padInput(x) if err != nil { return nil, err @@ -424,14 +423,16 @@ func (c *Conv) applyConv2D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) return nil, err } - for h := 0; h < paddedX.Shape()[2]; h += strideHSize { - dimHOutputIdx := h / strideHSize + // Loop over all 2D subImages of the input image and compute the convolution + // for that subImage. Store the result at the right place in the output tensor. + for h := 0; h < paddedX.Shape()[2]; h += c.strides[0] { + dimHOutputIdx := h / c.strides[0] if dimHOutputIdx >= outputHDim { continue } - for w := 0; w < paddedX.Shape()[2]; w += strideWSize { - dimWOutputIdx := w / strideWSize + for w := 0; w < paddedX.Shape()[2]; w += c.strides[1] { + dimWOutputIdx := w / c.strides[1] if dimWOutputIdx >= outputWDim { continue } @@ -441,10 +442,7 @@ func (c *Conv) applyConv2D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) return nil, err } - copiedSubKernel := subKernel.Materialize() - copiedSubImage := subImage.Materialize() - - convResult, err := tensor.Mul(copiedSubImage, copiedSubKernel) + convResult, err := tensor.Mul(subImage.Materialize(), subKernel.Materialize()) if err != nil { return nil, err } @@ -463,31 +461,41 @@ func (c *Conv) applyConv2D(x, kernel, bias tensor.Tensor) (tensor.Tensor, error) } } - if bias != nil { - err := bias.Reshape(1, bias.Shape()[0], 1, 1) - if err != nil { - return nil, err - } + return out, nil +} - out, bias, err = ops.MultidirectionalBroadcast(out, bias) - if err != nil { - return nil, err - } +// getOutputShape calculates the shape of the output tensor resulting from +// the convolution operation between `x` and `kernel`. +// `x` has shape [N, C, H, W, ...] and `kernel` has shape [M, C, H, W, ...]. +// The output shape will be [N, M, newH, newW, ...], where values like `newH` +// are calculated based on the input shape, kernel size, padding and strides. +func (c *Conv) getOutputShape(x, kernel tensor.Tensor) tensor.Shape { + outputShape := make([]int, len(x.Shape())) - out, err = tensor.Add(out, bias) - if err != nil { - return nil, err - } + outputShape[0] = x.Shape()[0] + outputShape[1] = kernel.Shape()[0] + + nSpatialDims := len(x.Shape()) - nNonSpatialDims + for i := 0; i < nSpatialDims; i++ { + inputDim := x.Shape()[nNonSpatialDims+i] + kernelDim := c.kernelShape[i] + outputShape[nNonSpatialDims+i] = ((inputDim - kernelDim + c.pads[i] + c.pads[i+nSpatialDims]) / c.strides[i]) + 1 } - return out, nil + return outputShape } +// padInput pads the input with zeros according to the `pads` attribute. +// The pad attribute specifies how many zeros should be added before and +// after the values in that specific dimension. +// Please note that according to ONNX specs, the `pads` attributes is an +// array with pads as [x1_begin, x2_begin, ..., x1_after, x2_after]. +// This method achieves padding by concatting tensors with zero values +// before and after each spatial dimension of the input tensor `x`. func (c *Conv) padInput(x tensor.Tensor) (tensor.Tensor, error) { var err error - nSpatialDims := len(x.Shape()[2:]) - nNonSpatialDims := 2 + nSpatialDims := len(x.Shape()[nNonSpatialDims:]) for i := 0; i < nSpatialDims; i++ { if c.pads[i] != 0 { @@ -513,7 +521,6 @@ func (c *Conv) padInput(x tensor.Tensor) (tensor.Tensor, error) { return nil, err } } - } return x, nil @@ -540,3 +547,27 @@ func (c *Conv) getSubImage(x tensor.Tensor, batchIdx int, startSpatialCoords ... return x.Slice(slices...) } + +// addBias adds a bias to the output of the convolution. It reshapes the +// bias such that it can be broadcasted, and then is added to the output +// tensor. +func (c *Conv) addBias(out, bias tensor.Tensor) (tensor.Tensor, error) { + biasShape := make([]int, len(out.Shape())) + for i := 0; i < len(out.Shape()); i++ { + biasShape[i] = 1 + } + + biasShape[1] = bias.Shape()[0] + + err := bias.Reshape(biasShape...) + if err != nil { + return nil, err + } + + out, bias, err = ops.MultidirectionalBroadcast(out, bias) + if err != nil { + return nil, err + } + + return tensor.Add(out, bias) +} diff --git a/ops_test.go b/ops_test.go index adc703a..75ae00f 100644 --- a/ops_test.go +++ b/ops_test.go @@ -107,6 +107,7 @@ type ONNXTestCase struct { func TestOps(t *testing.T) { var runnedTests []string + opNames := opset13.GetOpNames() for _, opName := range opNames { tests, err := getTestCasesForOp(opName) @@ -127,6 +128,7 @@ func TestOps(t *testing.T) { runnedTests = append(runnedTests, test.name) } } + sort.Strings(expectedTests) sort.Strings(runnedTests) assert.Equal(t, expectedTests, runnedTests) @@ -146,6 +148,7 @@ func getTestCasesForOp(opName string) ([]*ONNXTestCase, error) { } var tests []*ONNXTestCase + for _, testFolder := range testFolders { if shouldRunTest(testFolder, opFilter) { testcase, err := getTestCase(fmt.Sprintf("./test_data/%v", testFolder)) @@ -174,6 +177,7 @@ func shouldRunTest(folder, opFilter string) bool { return true } } + return false } @@ -186,6 +190,7 @@ func getTestCase(folder string) (*ONNXTestCase, error) { } basePath := fmt.Sprintf("%v/test_data_set_0", folder) + inputs, err := readTestTensors(basePath, "input", model.mp.Graph.GetInput()) if err != nil { return nil, err @@ -199,6 +204,7 @@ func getTestCase(folder string) (*ONNXTestCase, error) { testcase.model = model testcase.inputs = inputs testcase.outputs = outputs + return testcase, nil } @@ -229,8 +235,8 @@ func readTestTensors(basePath, baseFile string, inputs []*onnx.ValueInfoProto) ( tensors := make(Tensors) for i := 0; i < len(inputs); i++ { - filePath := fmt.Sprintf("%v/%v_%d.pb", basePath, baseFile, i) + bytesInput, err := ioutil.ReadFile(filePath) if err != nil { return nil, err diff --git a/sample_models/onnx_models/mnist-8-opset13.onnx b/sample_models/onnx_models/mnist-8-opset13.onnx new file mode 100644 index 0000000..5258a6b --- /dev/null +++ b/sample_models/onnx_models/mnist-8-opset13.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6267e75ad19e51ad643554f861f21fc76bcb54b625074a845ccf329c465bad6 +size 26454 diff --git a/sample_models/requirements.txt b/sample_models/requirements.txt index c72c959..3b70550 100644 --- a/sample_models/requirements.txt +++ b/sample_models/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.21.2 -scikit-learn==0.24.2 -skl2onnx==1.9.2 +numpy==1.26.1 +scikit-learn==1.3.1 +skl2onnx==1.15.0 torch==1.9.0 From e9c310f4d42cc5646c18b2e0de77a149c819db6f Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Fri, 20 Oct 2023 18:24:57 +0200 Subject: [PATCH 12/18] Add mnist model --- .gitattributes | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 3fe60cc..0000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -sample_models/onnx_models/mnist-8-opset13.onnx filter=lfs diff=lfs merge=lfs -text From 16211d07e92aad3659effc2880173a7e137b1320 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Sat, 21 Oct 2023 21:29:48 +0200 Subject: [PATCH 13/18] Added tests + bugfixes in conv --- ops/opset13/conv.go | 37 +++- ops/opset13/conv_test.go | 442 +++++++++++++++++++++++++++++++++++++++ ops/unidir_broadcast.go | 1 - ops/validate_inputs.go | 1 + 4 files changed, 473 insertions(+), 8 deletions(-) create mode 100644 ops/opset13/conv_test.go diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index eb1bebf..dd2109c 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -17,6 +17,10 @@ const ( Valid AutoPadSetting = "VALID" ) +// The number of non spatial dimensions inputs and kernels will always have. +// For input tensors, the first dimension will be the batch size. +// For kernel tensors, the first dimension will be the number of kernels. +// For all tensors, the second dimension will be the number of channels. const nNonSpatialDims = 2 // Conv represents the ONNX conv operator. @@ -359,14 +363,16 @@ func (c *Conv) applyConv1D(x, kernel tensor.Tensor) (tensor.Tensor, error) { for batchIdx := 0; batchIdx < nBatches; batchIdx++ { for kernelIdx := 0; kernelIdx < nKernels; kernelIdx++ { - subKernel, err := kernel.Slice(ops.NewSlicer(kernelIdx, kernelIdx+1)) + subKernelView, err := kernel.Slice(ops.NewSlicer(kernelIdx, kernelIdx+1)) if err != nil { return nil, err } + subKernel := subKernelView.Materialize() + for h := 0; h < paddedX.Shape()[2]; h += strideSize { dimHOutputIdx := h / strideSize - if dimHOutputIdx > outputHDim { + if dimHOutputIdx >= outputHDim { continue } @@ -375,6 +381,11 @@ func (c *Conv) applyConv1D(x, kernel tensor.Tensor) (tensor.Tensor, error) { return nil, err } + subImage, subKernel, err = ops.UnidirectionalBroadcast(subImage, subKernel) + if err != nil { + return nil, err + } + convResult, err := tensor.Mul(subImage, subKernel) if err != nil { return nil, err @@ -418,11 +429,13 @@ func (c *Conv) applyConv2D(x, kernel tensor.Tensor) (tensor.Tensor, error) { for batchIdx := 0; batchIdx < nBatches; batchIdx++ { for kernelIdx := 0; kernelIdx < nKernels; kernelIdx++ { - subKernel, err := kernel.Slice(ops.NewSlicer(kernelIdx, kernelIdx+1)) + subKernelView, err := kernel.Slice(ops.NewSlicer(kernelIdx, kernelIdx+1)) if err != nil { return nil, err } + subKernel := subKernelView.Materialize() + // Loop over all 2D subImages of the input image and compute the convolution // for that subImage. Store the result at the right place in the output tensor. for h := 0; h < paddedX.Shape()[2]; h += c.strides[0] { @@ -442,7 +455,12 @@ func (c *Conv) applyConv2D(x, kernel tensor.Tensor) (tensor.Tensor, error) { return nil, err } - convResult, err := tensor.Mul(subImage.Materialize(), subKernel.Materialize()) + subImage, subKernel, err = ops.UnidirectionalBroadcast(subImage, subKernel) + if err != nil { + return nil, err + } + + convResult, err := tensor.Mul(subImage, subKernel) if err != nil { return nil, err } @@ -529,7 +547,7 @@ func (c *Conv) padInput(x tensor.Tensor) (tensor.Tensor, error) { // getSubImage returns a the subimage for a specific example in the batch, based on the // kernel shape and the given start coordinates. The resulting sub image will be of // shape [1, C, kernelShape[0], kernelShape[1], ...]. -func (c *Conv) getSubImage(x tensor.Tensor, batchIdx int, startSpatialCoords ...int) (tensor.View, error) { +func (c *Conv) getSubImage(x tensor.Tensor, batchIdx int, startSpatialCoords ...int) (tensor.Tensor, error) { if len(startSpatialCoords) != len(c.kernelShape) { return nil, fmt.Errorf("expected the coordinates to have the same number of dimensions as the kernel") } @@ -545,7 +563,12 @@ func (c *Conv) getSubImage(x tensor.Tensor, batchIdx int, startSpatialCoords ... slices = append(slices, ops.NewSlicer(dimStartIdx, dimStartIdx+dimKernelSize)) } - return x.Slice(slices...) + subImage, err := x.Slice(slices...) + if err != nil { + return nil, err + } + + return subImage.Materialize(), nil } // addBias adds a bias to the output of the convolution. It reshapes the @@ -564,7 +587,7 @@ func (c *Conv) addBias(out, bias tensor.Tensor) (tensor.Tensor, error) { return nil, err } - out, bias, err = ops.MultidirectionalBroadcast(out, bias) + out, bias, err = ops.UnidirectionalBroadcast(out, bias) if err != nil { return nil, err } diff --git a/ops/opset13/conv_test.go b/ops/opset13/conv_test.go new file mode 100644 index 0000000..1f8b575 --- /dev/null +++ b/ops/opset13/conv_test.go @@ -0,0 +1,442 @@ +package opset13 + +import ( + "fmt" + "testing" + + "github.com/advancedclimatesystems/gonnx/onnx" + "github.com/advancedclimatesystems/gonnx/ops" + "github.com/stretchr/testify/assert" + "gorgonia.org/tensor" +) + +func TestConvInit(t *testing.T) { + c := &Conv{} + err := c.Init(Conv2DOnnxAttributeProtoFixture()) + + assert.Nil(t, err) + + var expectedAutopad AutoPadSetting = "VALID" + + assert.Equal(t, expectedAutopad, c.autoPad) + assert.Equal(t, []int{1, 1}, c.dilations) + assert.Equal(t, []int{2, 2}, c.kernelShape) + assert.Equal(t, []int{1, 2}, c.pads) + assert.Equal(t, []int{1, 1}, c.strides) +} + +func TestConvInitUnsupported(t *testing.T) { + c := &Conv{} + err := c.Init(ConvUnsupportedOnnxAttributeProtoFixture()) + + assert.Equal( + t, + err, + fmt.Errorf( + ops.UnsupportedAttrErrTemplate, + c, + "group", + ), + ) +} + +func TestConv(t *testing.T) { + tests := []struct { + conv *Conv + shapes [][]int + backings [][]float32 + expectedShape tensor.Shape + expected []float32 + }{ + // Test 1D Convolution. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{}, + group: 1, + kernelShape: []int{3}, + pads: []int{0, 0}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 6}, {1, 1, 3}}, + [][]float32{{0, 1, 2, 3, 4, 5}, {1, 1, 1}}, + []int{1, 1, 4}, + []float32{3, 6, 9, 12}, + }, + // Test 2D Convolution. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 3, 3}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1}}, + []int{1, 1, 2, 2}, + []float32{8, 12, 20, 24}, + }, + // Test SAME_LOWER autopad setting. + { + &Conv{ + autoPad: "SAME_LOWER", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 3, 3}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1}}, + []int{1, 1, 3, 3}, + []float32{0, 1, 3, 3, 8, 12, 9, 20, 24}, + }, + // Test SAME_UPPER autopad setting. + { + &Conv{ + autoPad: "SAME_UPPER", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 3, 3}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1}}, + []int{1, 1, 3, 3}, + []float32{8, 12, 7, 20, 24, 13, 13, 15, 8}, + }, + // Test VALID autopad setting. + { + &Conv{ + autoPad: "VALID", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 3, 3}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1}}, + []int{1, 1, 3, 3}, + []float32{8, 12, 7, 20, 24, 13, 13, 15, 8}, + }, + // Test dilation attribute. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{2, 2}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 4, 4}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, {1, 1, 1, 1}}, + []int{1, 1, 2, 2}, + []float32{20, 24, 36, 40}, + }, + // Test pads attribute. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{1, 1}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{1, 1, 2, 2}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 2, 2}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3}, {1, 1, 1, 1}}, + []int{1, 1, 4, 4}, + []float32{0, 1, 1, 0, 2, 6, 4, 0, 2, 5, 3, 0, 0, 0, 0, 0}, + }, + // Test strides attribute. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{2, 2}, + }, + [][]int{{1, 1, 4, 4}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, {1, 1, 1, 1}}, + []int{1, 1, 2, 2}, + []float32{10, 18, 42, 50}, + }, + // Test batch dimension. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{1, 1}, + }, + [][]int{{2, 1, 3, 3}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}, {1, 1, 1, 1}}, + []int{2, 1, 2, 2}, + []float32{8, 12, 20, 24, 44, 48, 56, 60}, + }, + // Test 2D convolution with multiple channels. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{1, 1}, + }, + [][]int{{1, 2, 3, 3}, {1, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}, {1, 1, 1, 1}}, + []int{1, 1, 2, 2}, + []float32{52, 60, 76, 84}, + }, + // Test multiple kernels. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 3, 3}, {2, 1, 2, 2}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1, 2, 2, 2, 2}}, + []int{1, 2, 2, 2}, + []float32{8, 12, 20, 24, 16, 24, 40, 48}, + }, + // Test bias. + { + &Conv{ + autoPad: "NOTSET", + dilations: []int{}, + group: 1, + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{1, 1}, + }, + [][]int{{1, 1, 3, 3}, {1, 1, 2, 2}, {1}}, + [][]float32{{0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1}, {0.5}}, + []int{1, 1, 2, 2}, + []float32{8.5, 12.5, 20.5, 24.5}, + }, + } + + for _, test := range tests { + inputs := []tensor.Tensor{ + ops.TensorWithBackingFixture(test.backings[0], test.shapes[0]...), + ops.TensorWithBackingFixture(test.backings[1], test.shapes[1]...), + } + + if len(test.backings) == 3 { + inputs = append(inputs, ops.TensorWithBackingFixture(test.backings[2], test.shapes[2]...)) + } + + res, err := test.conv.Apply(inputs) + assert.Nil(t, err) + + assert.Equal(t, test.expectedShape, res[0].Shape()) + assert.Equal(t, test.expected, res[0].Data()) + } +} + +func TestInputValidationConv(t *testing.T) { + tests := []struct { + inputs []tensor.Tensor + err error + }{ + { + []tensor.Tensor{ + ops.TensorWithBackingFixture([]float32{1, 2}, 2), + ops.TensorWithBackingFixture([]float32{3, 4}, 2), + nil, + }, + nil, + }, + { + []tensor.Tensor{ + ops.TensorWithBackingFixture([]float64{1, 2}, 2), + ops.TensorWithBackingFixture([]float64{3, 4}, 2), + ops.TensorWithBackingFixture([]float64{5, 6}, 2), + }, + nil, + }, + { + []tensor.Tensor{ + ops.TensorWithBackingFixture([]int{1, 2}, 2), + }, + fmt.Errorf("conv operator: expected 2-3 input tensors, got 1"), + }, + { + []tensor.Tensor{ + ops.TensorWithBackingFixture([]int{1, 2}, 2), + ops.TensorWithBackingFixture([]int{3, 4}, 2), + }, + fmt.Errorf("conv operator: input 0 does not allow type int"), + }, + } + + for _, test := range tests { + conv := &Conv{} + validated, err := conv.ValidateInputs(test.inputs) + + assert.Equal(t, test.err, err) + + if test.err == nil { + assert.Equal(t, test.inputs, validated) + } + } +} + +func TestSetDefaultDilations(t *testing.T) { + c := &Conv{} + x := ops.TensorWithBackingFixture([]float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, 1, 1, 3, 3) + + c.setDefaultDilations(x) + + assert.Equal(t, []int{1, 1}, c.dilations) +} + +func TestSetKernelShape(t *testing.T) { + c := &Conv{} + kernel := ops.TensorWithBackingFixture([]float32{0, 1, 2, 3}, 1, 1, 2, 2) + + c.setKernelShape(kernel) + + assert.Equal(t, []int{2, 2}, c.kernelShape) +} + +func TestSetDefaultPaddings(t *testing.T) { + c := &Conv{} + x := ops.TensorWithBackingFixture([]float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, 1, 1, 3, 3) + + c.setDefaultPaddings(x) + + assert.Equal(t, []int{0, 0, 0, 0}, c.pads) +} + +func TestSetDefaultStrides(t *testing.T) { + c := &Conv{} + x := ops.TensorWithBackingFixture([]float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, 1, 1, 3, 3) + + c.setDefaultStrides(x) + + assert.Equal(t, []int{1, 1}, c.strides) +} + +func TestSetPaddingWithAutoPad(t *testing.T) { + x := ops.TensorWithBackingFixture([]float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, 1, 1, 3, 3) + + tests := []struct { + setting AutoPadSetting + expectedPads []int + }{ + {"NOTSET", []int{0, 0, 0, 0}}, + {"SAME_LOWER", []int{1, 1, 0, 0}}, + {"SAME_UPPER", []int{0, 0, 1, 1}}, + {"VALID", []int{0, 0, 1, 1}}, + } + + for _, test := range tests { + conv := &Conv{ + autoPad: test.setting, + pads: []int{0, 0, 0, 0}, + kernelShape: []int{2, 2}, + strides: []int{1, 1}, + } + conv.setPaddingWithAutoPad(x) + + assert.Equal(t, test.expectedPads, conv.pads) + } +} + +func TestGetDilatedKernel(t *testing.T) { + tests := []struct { + dilations []int + kernelShape []int + kernelBacking []float32 + expectedShape tensor.Shape + expectedBacking []float32 + }{ + { + []int{1}, + []int{1, 1, 3}, + []float32{1, 1, 1}, + []int{1, 1, 3}, + []float32{1, 1, 1}, + }, + { + []int{2}, + []int{1, 1, 3}, + []float32{1, 1, 1}, + []int{1, 1, 5}, + []float32{1, 0, 1, 0, 1}, + }, + { + []int{2, 1}, + []int{1, 1, 2, 2}, + []float32{1, 1, 1, 1}, + []int{1, 1, 3, 2}, + []float32{1, 1, 0, 0, 1, 1}, + }, + { + []int{1, 2}, + []int{1, 1, 2, 2}, + []float32{1, 1, 1, 1}, + []int{1, 1, 2, 3}, + []float32{1, 0, 1, 1, 0, 1}, + }, + { + []int{2, 2}, + []int{1, 1, 3, 3}, + []float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, + []int{1, 1, 5, 5}, + []float32{0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3, 0, 4, 0, 5, 0, 0, 0, 0, 0, 6, 0, 7, 0, 8}, + }, + { + []int{3, 2}, + []int{1, 1, 2, 3}, + []float32{1, 2, 3, 4, 5, 6}, + []int{1, 1, 4, 5}, + []float32{1, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 6}, + }, + } + + for _, test := range tests { + conv := &Conv{ + dilations: test.dilations, + kernelShape: []int{2, 2}, + } + kernel := ops.TensorWithBackingFixture(test.kernelBacking, test.kernelShape...) + + dilatedKernel, err := conv.getDilatedKernel(kernel) + assert.Nil(t, err) + + assert.Equal(t, test.expectedShape, dilatedKernel.Shape()) + assert.Equal(t, test.expectedBacking, dilatedKernel.Data()) + } +} + +func Conv2DOnnxAttributeProtoFixture() []*onnx.AttributeProto { + return []*onnx.AttributeProto{ + {Name: "auto_pad", S: []byte("VALID")}, + {Name: "dilations", Ints: []int64{1, 1}}, + {Name: "kernel_shape", Ints: []int64{2, 2}}, + {Name: "pads", Ints: []int64{1, 2}}, + {Name: "strides", Ints: []int64{1, 1}}, + } +} + +func ConvUnsupportedOnnxAttributeProtoFixture() []*onnx.AttributeProto { + return []*onnx.AttributeProto{ + {Name: "group", I: 2}, + } +} diff --git a/ops/unidir_broadcast.go b/ops/unidir_broadcast.go index 41b35d5..702404c 100644 --- a/ops/unidir_broadcast.go +++ b/ops/unidir_broadcast.go @@ -8,7 +8,6 @@ import ( // UnidirectionalBroadcast tries to broadcast tensor B to tensor A according to the ONNX standards. func UnidirectionalBroadcast(A, B tensor.Tensor) (tensor.Tensor, tensor.Tensor, error) { - reshapedB, err := reshapeTensorsForUnidirBroadcast(A, B) if err != nil { return nil, nil, fmt.Errorf(UnidirBroadcastErrTemplate, A.Shape(), B.Shape()) diff --git a/ops/validate_inputs.go b/ops/validate_inputs.go index 8b954cb..caa862b 100644 --- a/ops/validate_inputs.go +++ b/ops/validate_inputs.go @@ -62,6 +62,7 @@ func padInputs(inputs []tensor.Tensor, length int) []tensor.Tensor { for len(inputs) < length { inputs = append(inputs, nil) } + return inputs } From 3dd5eca398191a05e7f8842966e81ce47e1b3d44 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Sun, 22 Oct 2023 08:06:08 +0200 Subject: [PATCH 14/18] Full coverage --- ops/opset13/conv.go | 2 +- ops/opset13/conv_test.go | 250 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+), 1 deletion(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index dd2109c..8d870b1 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -546,7 +546,7 @@ func (c *Conv) padInput(x tensor.Tensor) (tensor.Tensor, error) { // getSubImage returns a the subimage for a specific example in the batch, based on the // kernel shape and the given start coordinates. The resulting sub image will be of -// shape [1, C, kernelShape[0], kernelShape[1], ...]. +// shape [C, kernelShape[0], kernelShape[1], ...]. func (c *Conv) getSubImage(x tensor.Tensor, batchIdx int, startSpatialCoords ...int) (tensor.Tensor, error) { if len(startSpatialCoords) != len(c.kernelShape) { return nil, fmt.Errorf("expected the coordinates to have the same number of dimensions as the kernel") diff --git a/ops/opset13/conv_test.go b/ops/opset13/conv_test.go index 1f8b575..ba072e8 100644 --- a/ops/opset13/conv_test.go +++ b/ops/opset13/conv_test.go @@ -425,6 +425,256 @@ func TestGetDilatedKernel(t *testing.T) { } } +func TestGetOutputShape(t *testing.T) { + tests := []struct { + conv *Conv + xShape []int + xBacking []float32 + kernelShape []int + kernelBacking []float32 + expected tensor.Shape + }{ + { + &Conv{ + kernelShape: []int{3}, + pads: []int{0, 0}, + strides: []int{1}, + }, + []int{1, 1, 6}, + []float32{0, 1, 2, 3, 4, 5}, + []int{1, 1, 3}, + []float32{1, 1, 1}, + []int{1, 1, 4}, + }, + { + &Conv{ + kernelShape: []int{3}, + pads: []int{1, 2}, + strides: []int{2}, + }, + []int{1, 1, 6}, + []float32{0, 1, 2, 3, 4, 5}, + []int{1, 1, 3}, + []float32{1, 1, 1}, + []int{1, 1, 4}, + }, + { + &Conv{ + kernelShape: []int{2, 2}, + pads: []int{1, 2, 1, 2}, + strides: []int{2, 1}, + }, + []int{1, 1, 4, 4}, + []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + []int{1, 1, 2, 2}, + []float32{1, 1, 1, 1}, + []int{1, 1, 3, 7}, + }, + { + &Conv{ + kernelShape: []int{2, 2}, + pads: []int{0, 0, 0, 0}, + strides: []int{1, 1}, + }, + []int{1, 1, 4, 4}, + []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + []int{1, 1, 2, 2}, + []float32{1, 1, 1, 1}, + []int{1, 1, 3, 3}, + }, + } + + for _, test := range tests { + outputShape := test.conv.getOutputShape( + ops.TensorWithBackingFixture(test.xBacking, test.xShape...), + ops.TensorWithBackingFixture(test.kernelBacking, test.kernelShape...), + ) + + assert.Equal(t, test.expected, outputShape) + } +} + +func TestPadInput(t *testing.T) { + tests := []struct { + conv *Conv + xShape []int + xBacking []float32 + expectedShape tensor.Shape + expectedBacking []float32 + }{ + { + &Conv{ + pads: []int{0, 0}, + }, + []int{1, 1, 6}, + []float32{0, 1, 2, 3, 4, 5}, + []int{1, 1, 6}, + []float32{0, 1, 2, 3, 4, 5}, + }, + { + &Conv{ + pads: []int{1, 2}, + }, + []int{1, 1, 6}, + []float32{0, 1, 2, 3, 4, 5}, + []int{1, 1, 9}, + []float32{0, 0, 1, 2, 3, 4, 5, 0, 0}, + }, + { + &Conv{ + pads: []int{1, 1, 1, 1}, + }, + []int{1, 1, 2, 2}, + []float32{1, 2, 3, 4}, + []int{1, 1, 4, 4}, + []float32{0, 0, 0, 0, 0, 1, 2, 0, 0, 3, 4, 0, 0, 0, 0, 0}, + }, + { + &Conv{ + pads: []int{1, 0, 2, 0}, + }, + []int{1, 1, 2, 2}, + []float32{1, 2, 3, 4}, + []int{1, 1, 5, 2}, + []float32{0, 0, 1, 2, 3, 4, 0, 0, 0, 0}, + }, + } + + for _, test := range tests { + paddedX, err := test.conv.padInput( + ops.TensorWithBackingFixture(test.xBacking, test.xShape...), + ) + + assert.Nil(t, err) + assert.Equal(t, test.expectedShape, paddedX.Shape()) + assert.Equal(t, test.expectedBacking, paddedX.Data()) + } +} + +func TestGetSubImage(t *testing.T) { + tests := []struct { + conv *Conv + xShape []int + xBacking []float32 + batchIdx int + startSpatialCoords []int + expectedShape tensor.Shape + expectedBacking []float32 + }{ + { + &Conv{kernelShape: []int{2}}, + []int{1, 1, 3}, + []float32{0, 1, 2}, + 0, + []int{0}, + []int{1, 2}, + []float32{0, 1}, + }, + { + &Conv{kernelShape: []int{2}}, + []int{1, 2, 3}, + []float32{0, 1, 2, 3, 4, 5}, + 0, + []int{0}, + []int{2, 2}, + []float32{0, 1, 3, 4}, + }, + { + &Conv{kernelShape: []int{2, 2}}, + []int{1, 1, 3, 3}, + []float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, + 0, + []int{0, 0}, + []int{1, 2, 2}, + []float32{0, 1, 3, 4}, + }, + { + &Conv{kernelShape: []int{2, 2}}, + []int{1, 1, 3, 3}, + []float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, + 0, + []int{1, 1}, + []int{1, 2, 2}, + []float32{4, 5, 7, 8}, + }, + { + &Conv{kernelShape: []int{2}}, + []int{2, 1, 3}, + []float32{0, 1, 2, 3, 4, 5}, + 1, + []int{1}, + []int{1, 2}, + []float32{4, 5}, + }, + } + + for _, test := range tests { + subImage, err := test.conv.getSubImage( + ops.TensorWithBackingFixture(test.xBacking, test.xShape...), + test.batchIdx, + test.startSpatialCoords..., + ) + + assert.Nil(t, err) + assert.Equal(t, test.expectedShape, subImage.Shape()) + assert.Equal(t, test.expectedBacking, subImage.Data()) + } +} + +func TestAddBias(t *testing.T) { + tests := []struct { + conv *Conv + outShape []int + outBacking []float32 + biasShape []int + biasBacking []float32 + expected []float32 + }{ + { + &Conv{}, + []int{1, 1, 3}, + []float32{0, 1, 2}, + []int{1}, + []float32{0.5}, + []float32{0.5, 1.5, 2.5}, + }, + { + &Conv{}, + []int{1, 1, 3, 3}, + []float32{0, 1, 2, 3, 4, 5, 6, 7, 8}, + []int{1}, + []float32{0.5}, + []float32{0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5}, + }, + { + &Conv{}, + []int{1, 2, 2, 2}, + []float32{0, 1, 2, 3, 4, 5, 6, 7}, + []int{2}, + []float32{-1, 1}, + []float32{-1, 0, 1, 2, 5, 6, 7, 8}, + }, + { + &Conv{}, + []int{2, 2, 2, 2}, + []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + []int{2}, + []float32{-1, 1}, + []float32{-1, 0, 1, 2, 5, 6, 7, 8, 7, 8, 9, 10, 13, 14, 15, 16}, + }, + } + + for _, test := range tests { + out, err := test.conv.addBias( + ops.TensorWithBackingFixture(test.outBacking, test.outShape...), + ops.TensorWithBackingFixture(test.biasBacking, test.biasShape...), + ) + + assert.Nil(t, err) + assert.Equal(t, test.expected, out.Data()) + } +} + func Conv2DOnnxAttributeProtoFixture() []*onnx.AttributeProto { return []*onnx.AttributeProto{ {Name: "auto_pad", S: []byte("VALID")}, From efeaa1a6baa9942091fba86385b754e23d8b5b95 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Sun, 22 Oct 2023 08:12:01 +0200 Subject: [PATCH 15/18] Remove unnecessary if --- ops/opset13/conv.go | 6 +----- ops/opset13/conv_test.go | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index 8d870b1..384ca90 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -96,11 +96,7 @@ func (c *Conv) Init(attributes []*onnx.AttributeProto) error { func (c *Conv) Apply(inputs []tensor.Tensor) ([]tensor.Tensor, error) { x := inputs[0] kernel := inputs[1] - - var bias tensor.Tensor - if len(inputs) == c.GetMaxInputs() { - bias = inputs[2] - } + bias := inputs[2] if len(c.dilations) == 0 { c.setDefaultDilations(x) diff --git a/ops/opset13/conv_test.go b/ops/opset13/conv_test.go index ba072e8..970fcef 100644 --- a/ops/opset13/conv_test.go +++ b/ops/opset13/conv_test.go @@ -234,10 +234,11 @@ func TestConv(t *testing.T) { inputs := []tensor.Tensor{ ops.TensorWithBackingFixture(test.backings[0], test.shapes[0]...), ops.TensorWithBackingFixture(test.backings[1], test.shapes[1]...), + nil, } if len(test.backings) == 3 { - inputs = append(inputs, ops.TensorWithBackingFixture(test.backings[2], test.shapes[2]...)) + inputs[2] = ops.TensorWithBackingFixture(test.backings[2], test.shapes[2]...) } res, err := test.conv.Apply(inputs) From d37d6563d99ec83110006a400c5ce2091c6c81fb Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Sat, 11 Nov 2023 21:20:16 +0100 Subject: [PATCH 16/18] Kept division by 2 --- ops/opset13/conv.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index dea12fc..b620c4f 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -249,9 +249,11 @@ func (c *Conv) setPaddingWithAutoPad(x tensor.Tensor) { var padHead int if c.autoPad == "SAME_LOWER" { - padHead = (padNeeded + 1) / NPadsPerDim + // nolint + padHead = (padNeeded + 1) / 2 } else { - padHead = padNeeded / NPadsPerDim + //nolint + padHead = padNeeded / 2 } padTail := padNeeded - padHead From 0a2bae907390fb9e7e08b91e98c0195544556f96 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Fri, 24 Nov 2023 10:45:10 +0100 Subject: [PATCH 17/18] Fix last MR comment --- ops/opset13/conv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index 0d8d862..d762731 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -238,10 +238,10 @@ func (c *Conv) setPaddingWithAutoPad(x tensor.Tensor) { var padHead int if c.autoPad == SameLower { - // nolint + // nolint, as the division by two is literally division by two padHead = (padNeeded + 1) / 2 } else { - //nolint + // nolint, as the division by two is literally division by two padHead = padNeeded / 2 } From 72d5d9585a3595d01958e6285ba3ef04272e6d09 Mon Sep 17 00:00:00 2001 From: Swopper050 Date: Fri, 24 Nov 2023 10:47:36 +0100 Subject: [PATCH 18/18] Fix lint? --- ops/opset13/conv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ops/opset13/conv.go b/ops/opset13/conv.go index d762731..01a82d1 100644 --- a/ops/opset13/conv.go +++ b/ops/opset13/conv.go @@ -238,10 +238,10 @@ func (c *Conv) setPaddingWithAutoPad(x tensor.Tensor) { var padHead int if c.autoPad == SameLower { - // nolint, as the division by two is literally division by two + // nolint as the division by zero is literally division by two padHead = (padNeeded + 1) / 2 } else { - // nolint, as the division by two is literally division by two + // nolint as the division by two is literally division by two padHead = padNeeded / 2 }