diff --git a/csharp/.gitattributes b/csharp/.gitattributes new file mode 100644 index 0000000000000..d2ff52b121fc5 --- /dev/null +++ b/csharp/.gitattributes @@ -0,0 +1,36 @@ +############################################################################### +# Set default behavior to automatically normalize line endings. +############################################################################### +* text=auto + +############################################################################### +# Set default behavior for command prompt diff. +# +# This is need for earlier builds of msysgit that does not have it on by +# default for csharp files. +# Note: This is only used by command line +############################################################################### +#*.cs diff=csharp + +############################################################################### +# Set the merge driver for project and solution files +# +# Merging from the command prompt will add diff markers to the files if there +# are conflicts (Merging from VS is not affected by the settings below, in VS +# the diff markers are never inserted). Diff markers may cause the following +# file extensions to fail to load in VS. An alternative would be to treat +# these files as binary and thus will always conflict and require user +# intervention with every merge. To do so, just uncomment the entries below +############################################################################### +#*.sln merge=binary +#*.csproj merge=binary +#*.vbproj merge=binary +#*.vcxproj merge=binary +#*.vcproj merge=binary +#*.dbproj merge=binary +#*.fsproj merge=binary +#*.lsproj merge=binary +#*.wixproj merge=binary +#*.modelproj merge=binary +#*.sqlproj merge=binary +#*.wwaproj merge=binary \ No newline at end of file diff --git a/csharp/.gitignore b/csharp/.gitignore new file mode 100644 index 0000000000000..e5b411f791302 --- /dev/null +++ b/csharp/.gitignore @@ -0,0 +1,264 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Project-specific +artifacts/ \ No newline at end of file diff --git a/csharp/README.md b/csharp/README.md new file mode 100644 index 0000000000000..fc393c20c9082 --- /dev/null +++ b/csharp/README.md @@ -0,0 +1,150 @@ + + +# Apache Arrow + +An implementation of Arrow targeting .NET Standard. + +This implementation is under development and may not be suitable for use in production environments. + +# Implementation + +- Arrow 0.11 (specification) +- C# 7.2 +- .NET Standard 1.3 +- Asynchronous I/O +- Uses modern .NET runtime features such as **Span<T>**, **Memory<T>**, **MemoryManager<T>**, and **System.Buffers** primitives for memory allocation, memory storage, and fast serialization. +- Uses **Acyclic Visitor Pattern** for array types and arrays to facilitate serialization, record batch traversal, and format growth. + +# Known Issues + +- Can not read Arrow files containing dictionary batches, tensors, or tables. +- Can not easily modify allocation strategy without implementing a custom memory pool. All allocations are currently 64-byte aligned and padded to 8-bytes. +- Default memory allocation strategy uses an over-allocation strategy with pointer fixing, which results in significant memory overhead for small buffers. A buffer that requires a single byte for storage may be backed by an allocation of up to 64-bytes to satisfy alignment requirements. +- There are currently few builder APIs available for specific array types. Arrays must be built manually with an arrow buffer builder abstraction. +- FlatBuffer code generation is not included in the build process. +- Serialization implementation does not perform exhaustive validation checks during deserialization in every scenario. +- Throws exceptions with vague, inconsistent, or non-localized messages in many situations +- Throws exceptions that are non-specific to the Arrow implementation in some circumstances where it probably should (eg. does not throw ArrowException exceptions) +- Lack of code documentation +- Lack of usage examples +- Lack of comprehensive unit tests +- Lack of comprehensive benchmarks + +# Usage + + using System.Diagnostics; + using System.IO; + using System.Threading.Tasks; + using Apache.Arrow; + using Apache.Arrow.Ipc; + + public static async Task ReadArrowAsync(string filename) + { + using (var stream = File.OpenRead("test.arrow")) + using (var reader = new ArrowFileReader(stream)) + { + var recordBatch = await reader.ReadNextRecordBatchAsync(); + Debug.WriteLine("Read record batch with {0} column(s)", recordBatch.ColumnCount); + return recordBatch; + } + } + + +# Status + +## Memory Management + +- Allocations are 64-byte aligned and padded to 8-bytes. +- Allocations are automatically garbage collected + +## Arrays + +### Primitive Types + +- Int8, Int16, Int32, Int64 +- UInt8, UInt16, UInt32, UInt64 +- Float, Double +- Binary (variable-length) +- String (utf-8) +- Null + +### Parametric Types + +- Timestamp +- Date32 +- Date64 +- Time32 +- Time64 +- Binary (fixed-length) +- List + +### Type Metadata + +- Data Types +- Fields +- Schema + +### Serialization + +- File +- Stream + +## Not Implemented + +- Serialization + - Exhaustive validation + - Dictionary Batch + - Can not serialize or deserialize files or streams containing dictionary batches + - Dictionary Encoding + - Schema Metadata + - Schema Field Metadata +- Types + - Tensor + - Table +- Arrays + - Struct + - Union + - Dense + - Sparse + - Half-Float + - Decimal + - Dictionary +- Array Operations + - Equality / Comparison + - Casting + - Builders +- Compute + - There is currently no API available for a compute / kernel abstraction. + +# Build + + dotnet build + +# Docker Build + +Build from the Apache Arrow project root. + + docker build -f csharp/build/docker/Dockerfile . + +# Testing + + dotnet test test/Apache.Arrow.Tests + +All build artifacts are placed in the **artifacts** folder in the project root. diff --git a/csharp/build/Common.props b/csharp/build/Common.props new file mode 100644 index 0000000000000..9e7901d8a109f --- /dev/null +++ b/csharp/build/Common.props @@ -0,0 +1,8 @@ + + + ../../artifacts/$(AssemblyName) + + + + + \ No newline at end of file diff --git a/csharp/build/docker/Dockerfile b/csharp/build/docker/Dockerfile new file mode 100644 index 0000000000000..e6ad5741f3fa7 --- /dev/null +++ b/csharp/build/docker/Dockerfile @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM microsoft/dotnet/2.1-sdk + +ADD csharp /arrow/csharp +WORKDIR /arrow/csharp + +CMD dotnet build \ No newline at end of file diff --git a/csharp/src/Apache.Arrow/Apache.Arrow.csproj b/csharp/src/Apache.Arrow/Apache.Arrow.csproj new file mode 100644 index 0000000000000..adc21c9edc07b --- /dev/null +++ b/csharp/src/Apache.Arrow/Apache.Arrow.csproj @@ -0,0 +1,50 @@ + + + + + + netstandard1.3 + Apache + Apache Arrow library + 2018 Apache Software Foundation + https://fzcorp.visualstudio.com/digital-products + https://fzcorp.visualstudio.com/digital-products/_git/fz-arrow + git + apache arrow + Apache + 0.0.1 + + + + 7.2 + true + + + + true + 7.2 + + + + + + + + + + + + True + True + Resources.resx + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + diff --git a/csharp/src/Apache.Arrow/Arrays/Array.cs b/csharp/src/Apache.Arrow/Arrays/Array.cs new file mode 100644 index 0000000000000..a9609f20f1210 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/Array.cs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Runtime.CompilerServices; + +namespace Apache.Arrow +{ + public abstract class Array : IArrowArray + { + public ArrayData Data { get; } + + protected Array(ArrayData data) + { + Data = data ?? throw new ArgumentNullException(nameof(data)); + } + + public int Length => Data.Length; + + public int Offset => Data.Offset; + + public int NullCount => Data.NullCount; + + public Bitmap NullBitmap => Data.NullBitmap; + + public ArrowBuffer NullBitmapBuffer => Data.Buffers[0]; + + public virtual void Accept(IArrowArrayVisitor visitor) + { + Accept(this, visitor); + } + + public bool IsValid(int index) => + NullBitmapBuffer == null || NullBitmap.IsSet(index); + + public bool IsNull(int index) => !IsValid(index); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void Accept(T array, IArrowArrayVisitor visitor) + where T : class, IArrowArray + { + if (visitor is IArrowArrayVisitor v) + { + v.Visit(array); + } + else + { + visitor.Visit(array); + } + } + } +} \ No newline at end of file diff --git a/csharp/src/Apache.Arrow/Arrays/ArrayData.cs b/csharp/src/Apache.Arrow/Arrays/ArrayData.cs new file mode 100644 index 0000000000000..a8d745c66c150 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/ArrayData.cs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow +{ + public class ArrayData + { + public readonly IArrowType DataType; + public readonly int Length; + public readonly int NullCount; + public readonly int Offset; + public readonly ArrowBuffer[] Buffers; + public readonly ArrayData[] Children; + + public ArrowBuffer NullBitmapBuffer => Buffers[0]; + public Bitmap NullBitmap => NullBitmapBuffer; + + public ArrayData( + IArrowType dataType, + int length, int nullCount = 0, int offset = 0, + IEnumerable buffers = null, IEnumerable children = null) + { + DataType = dataType ?? NullType.Default; + Length = length; + NullCount = nullCount; + Offset = offset; + Buffers = buffers?.ToArray(); + Children = children?.ToArray(); + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs b/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs new file mode 100644 index 0000000000000..542871cd3f726 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + public class ArrowArrayFactory + { + private class FactoryTypeVisitor : + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor + { + private readonly ArrayData _data; + private IArrowArray _array; + + public FactoryTypeVisitor(ArrayData data) + { + _data = data; + } + + public IArrowArray CreateArray() + { + _data.DataType.Accept(this); + return _array; + } + + public void Visit(Int8Type type) => _array = new Int8Array(_data); + public void Visit(Int16Type type) => _array = new Int16Array(_data); + public void Visit(Int32Type type) => _array = new Int32Array(_data); + public void Visit(Int64Type type) => _array = new Int64Array(_data); + public void Visit(UInt8Type type) => _array = new UInt8Array(_data); + public void Visit(UInt16Type type) => _array = new UInt16Array(_data); + public void Visit(UInt32Type type) => _array = new UInt32Array(_data); + public void Visit(UInt64Type type) => _array = new UInt64Array(_data); + public void Visit(BooleanType type) => _array = new BooleanArray(_data); + public void Visit(FloatType type) => _array = new FloatArray(_data); + public void Visit(DoubleType type) => _array = new DoubleArray(_data); + public void Visit(StructType type) => _array = new StructArray(_data); + public void Visit(UnionType type) => _array = new UnionArray(_data); + public void Visit(ListType type) => _array = new ListArray(_data); + public void Visit(TimestampType type) => _array = new TimestampArray(_data); + public void Visit(BinaryType type) => _array = new BinaryArray(_data); + public void Visit(StringType type) => _array = new StringArray(_data); + + public void Visit(IArrowType type) + { + throw new NotImplementedException(); + } + } + + public static IArrowArray BuildArray(ArrayData data) + { + var visitor = new FactoryTypeVisitor(data); + var array = visitor.CreateArray(); + return array; + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs b/csharp/src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs new file mode 100644 index 0000000000000..fc56b6601f5e5 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/ArrowArrayVisitor.cs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Apache.Arrow +{ + public abstract class ArrowArrayVisitor : IArrowArrayVisitor + { + public virtual void Visit(IArrowArray array) { } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/BinaryArray.cs b/csharp/src/Apache.Arrow/Arrays/BinaryArray.cs new file mode 100644 index 0000000000000..cbe64dfc1bba2 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/BinaryArray.cs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Runtime.CompilerServices; + +namespace Apache.Arrow +{ + public class BinaryArray: Array + { + public BinaryArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Binary); + data.EnsureBufferCount(3); + } + + public BinaryArray(ArrowTypeId typeId, ArrayData data) + : base (data) + { + data.EnsureDataType(typeId); + data.EnsureBufferCount(3); + } + + public BinaryArray(IArrowType dataType, int length, + ArrowBuffer valueOffsetsBuffer, + ArrowBuffer dataBuffer, + ArrowBuffer nullBitmapBuffer, + int nullCount = 0, int offset = 0) + : this(new ArrayData(dataType, length, nullCount, offset, + new [] { nullBitmapBuffer, valueOffsetsBuffer, dataBuffer })) + { } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public ArrowBuffer ValueOffsetsBuffer => Data.Buffers[1]; + + public ArrowBuffer ValueBuffer => Data.Buffers[2]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetValueOffset(int index) + { + var offsets = ValueOffsetsBuffer.GetSpan(); + return offsets[Offset + index]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetValueLength(int index) + { + var offsets = ValueOffsetsBuffer.GetSpan(); + var offset = Offset + index; + return offsets[offset + 1] - offsets[offset]; + } + + public ReadOnlySpan GetValue(int index) + { + var offset = GetValueOffset(index); + var length = GetValueLength(index); + var values = ValueBuffer.GetSpan(); + + return values.Slice(offset, length); + } + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/BooleanArray.cs b/csharp/src/Apache.Arrow/Arrays/BooleanArray.cs new file mode 100644 index 0000000000000..f4197122e37c0 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/BooleanArray.cs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class BooleanArray: PrimitiveArray + { + public BooleanArray( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(BooleanType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public BooleanArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Boolean); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public bool? GetBoolean(int index) + { + if (IsNull(index)) + return null; + + var span = GetSpan(); + return BitUtility.GetBit(span, index); + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/Date32Array.cs b/csharp/src/Apache.Arrow/Arrays/Date32Array.cs new file mode 100644 index 0000000000000..a0ea7e2bf8eae --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/Date32Array.cs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + public class Date32Array: PrimitiveArray + { + public Date32Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Date32Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Date32Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Date32); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public DateTimeOffset? GetDate(int index) + { + var value = GetValue(index); + + const long millisecondsPerDay = 86_400_000; + + return value.HasValue + ? DateTimeOffset.FromUnixTimeMilliseconds(value.Value * millisecondsPerDay) + : default; + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/Date64Array.cs b/csharp/src/Apache.Arrow/Arrays/Date64Array.cs new file mode 100644 index 0000000000000..307cda0722696 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/Date64Array.cs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + public class Date64Array: PrimitiveArray + { + public Date64Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Date64Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Date64Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Date64); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public DateTimeOffset? GetDate(int index) + { + var value = GetValue(index); + + return value.HasValue + ? DateTimeOffset.FromUnixTimeMilliseconds(value.Value) + : default; + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/DoubleArray.cs b/csharp/src/Apache.Arrow/Arrays/DoubleArray.cs new file mode 100644 index 0000000000000..0c50e5a90618b --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/DoubleArray.cs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class DoubleArray : PrimitiveArray + { + public DoubleArray( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(DoubleType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public DoubleArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Double); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/FloatArray.cs b/csharp/src/Apache.Arrow/Arrays/FloatArray.cs new file mode 100644 index 0000000000000..b4c9e607a83cf --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/FloatArray.cs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class FloatArray : PrimitiveArray + { + public FloatArray( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(FloatType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public FloatArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Float); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/Int16Array.cs b/csharp/src/Apache.Arrow/Arrays/Int16Array.cs new file mode 100644 index 0000000000000..639c6d5c5e864 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/Int16Array.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int16Array : PrimitiveArray + { + public Int16Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int16Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int16Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int16); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/Int32Array.cs b/csharp/src/Apache.Arrow/Arrays/Int32Array.cs new file mode 100644 index 0000000000000..8bd0048d64193 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/Int32Array.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int32Array : PrimitiveArray + { + public Int32Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int32Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int32Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int32); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/Int64Array.cs b/csharp/src/Apache.Arrow/Arrays/Int64Array.cs new file mode 100644 index 0000000000000..d010a8dee3ed4 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/Int64Array.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int64Array : PrimitiveArray + { + public Int64Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int64Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int64Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int64); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/Int8Array.cs b/csharp/src/Apache.Arrow/Arrays/Int8Array.cs new file mode 100644 index 0000000000000..67e68bd0975ca --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/Int8Array.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class Int8Array : PrimitiveArray + { + public Int8Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(Int8Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public Int8Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Int8); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/ListArray.cs b/csharp/src/Apache.Arrow/Arrays/ListArray.cs new file mode 100644 index 0000000000000..e3872bceb0abb --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/ListArray.cs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class ListArray : Array + { + public IArrowArray Values { get; } + + public ArrowBuffer ValueOffsetsBuffer => Data.Buffers[1]; + + public ListArray(IArrowType dataType, int length, + ArrowBuffer valueOffsetsBuffer, IArrowArray values, + ArrowBuffer nullBitmapBuffer, int nullCount = 0, int offset = 0) + : this(new ArrayData(dataType, length, nullCount, offset, + new[] {nullBitmapBuffer, valueOffsetsBuffer}, new[] {values.Data})) + { + Values = values; + } + + public ListArray(ArrayData data) + : base(data) + { + data.EnsureBufferCount(2); + data.EnsureDataType(ArrowTypeId.List); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public int GetValueOffset(int index) + { + var span = ValueOffsetsBuffer.GetSpan(Offset); + return span[index]; + } + + public int GetValueLength(int index) + { + var span = ValueOffsetsBuffer.GetSpan(Offset); + return span[index + 1] - span[index]; + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/PrimitiveArray.cs b/csharp/src/Apache.Arrow/Arrays/PrimitiveArray.cs new file mode 100644 index 0000000000000..6dcb10333d2fd --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/PrimitiveArray.cs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using Apache.Arrow.Memory; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public abstract class PrimitiveArray : Array + where T : struct + { + + protected PrimitiveArray(ArrayData data) + : base(data) + { + data.EnsureBufferCount(2); + } + + public ArrowBuffer ValueBuffer => Data.Buffers[1]; + + public Span GetSpan() => ValueBuffer.GetSpan().Slice(0, Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T? GetValue(int index) + { + var span = GetSpan(); + return IsValid(index) ? span[index] : (T?) null; + } + + public IList ToList(bool includeNulls = false) + { + var span = GetSpan(); + var list = new List(span.Length); + + for (var i = 0; i < span.Length; i++) + { + var value = GetValue(i); + + if (value.HasValue) + { + list.Add(value.Value); + } + else + { + if (includeNulls) + { + list.Add(null); + } + } + } + + return list; + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/StringArray.cs b/csharp/src/Apache.Arrow/Arrays/StringArray.cs new file mode 100644 index 0000000000000..3c8f8c0599527 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/StringArray.cs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Runtime.InteropServices; +using System.Text; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class StringArray: BinaryArray + { + public StringArray(ArrayData data) + : base(ArrowTypeId.String, data) { } + + public StringArray(int length, + ArrowBuffer valueOffsetsBuffer, + ArrowBuffer dataBuffer, + ArrowBuffer nullBitmapBuffer, + int nullCount = 0, int offset = 0) + : this(new ArrayData(StringType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueOffsetsBuffer, dataBuffer })) + { } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public string GetString(int index, Encoding encoding = default) + { + encoding = encoding ?? Encoding.UTF8; + + var value = GetValue(index); + + unsafe + { + fixed (byte* data = &MemoryMarshal.GetReference(value)) + return encoding.GetString(data, value.Length); + } + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/StructArray.cs b/csharp/src/Apache.Arrow/Arrays/StructArray.cs new file mode 100644 index 0000000000000..a9dc5bc20231d --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/StructArray.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow +{ + public class StructArray : Array + { + private readonly List _fields; + + public IEnumerable Fields => _fields; + + public StructArray( + IArrowType dataType, int length, + IEnumerable children, + ArrowBuffer nullBitmapBuffer, int nullCount = 0, int offset = 0) + : this(new ArrayData( + dataType, length, nullCount, offset, new[] { nullBitmapBuffer }, + children.Select(child => child.Data))) + { } + + public StructArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Struct); + + _fields = new List(); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/TimestampArray.cs b/csharp/src/Apache.Arrow/Arrays/TimestampArray.cs new file mode 100644 index 0000000000000..174f6500a1058 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/TimestampArray.cs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.IO; + +namespace Apache.Arrow +{ + public class TimestampArray: PrimitiveArray + { + public TimestampArray( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(TimestampType.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public TimestampArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Timestamp); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + public DateTimeOffset? GetTimestamp(int index) + { + var span = GetSpan(); + + if (IsNull(index)) + { + return null; + } + + var value = span[index]; + var type = Data.DataType as TimestampType; + + switch (type.Unit) + { + case TimeUnit.Nanosecond: + return DateTimeOffset.FromUnixTimeMilliseconds(value / 1000000); + case TimeUnit.Microsecond: + return DateTimeOffset.FromUnixTimeMilliseconds(value / 1000); + case TimeUnit.Millisecond: + return DateTimeOffset.FromUnixTimeMilliseconds(value); + case TimeUnit.Second: + return DateTimeOffset.FromUnixTimeSeconds(value); + default: + throw new InvalidDataException( + string.Format("Unsupported timestamp unit <{0}>", type.Unit)); + } + } + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/UInt16Array.cs b/csharp/src/Apache.Arrow/Arrays/UInt16Array.cs new file mode 100644 index 0000000000000..8ffe3c166c906 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/UInt16Array.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt16Array : PrimitiveArray + { + public UInt16Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt16Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public UInt16Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt16); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + } + +} diff --git a/csharp/src/Apache.Arrow/Arrays/UInt32Array.cs b/csharp/src/Apache.Arrow/Arrays/UInt32Array.cs new file mode 100644 index 0000000000000..0321ea59af7be --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/UInt32Array.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt32Array : PrimitiveArray + { + public UInt32Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt32Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public UInt32Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt32); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/UInt64Array.cs b/csharp/src/Apache.Arrow/Arrays/UInt64Array.cs new file mode 100644 index 0000000000000..09b1e6d37a20c --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/UInt64Array.cs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt64Array : PrimitiveArray + { + public UInt64Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt64Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) + { } + + public UInt64Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt64); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/UInt8Array.cs b/csharp/src/Apache.Arrow/Arrays/UInt8Array.cs new file mode 100644 index 0000000000000..e4506b5c56c15 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/UInt8Array.cs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public class UInt8Array : PrimitiveArray + { + public UInt8Array( + ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer, + int length, int nullCount, int offset) + : this(new ArrayData(UInt8Type.Default, length, nullCount, offset, + new[] { nullBitmapBuffer, valueBuffer })) { } + + public UInt8Array(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.UInt8); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/Arrays/UnionArray.cs b/csharp/src/Apache.Arrow/Arrays/UnionArray.cs new file mode 100644 index 0000000000000..7ba7f9f5b8e27 --- /dev/null +++ b/csharp/src/Apache.Arrow/Arrays/UnionArray.cs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + public class UnionArray: Array + { + public UnionType Type => Data.DataType as UnionType; + + public UnionMode Mode => Type.Mode; + + public ArrowBuffer TypeBuffer => Data.Buffers[1]; + + public ArrowBuffer ValueOffsetBuffer => Data.Buffers[2]; + + public ReadOnlySpan TypeIds => TypeBuffer.GetSpan(); + + public ReadOnlySpan ValueOffsets => ValueOffsetBuffer.GetSpan(); + + public UnionArray(ArrayData data) + : base(data) + { + data.EnsureDataType(ArrowTypeId.Union); + data.EnsureBufferCount(3); + } + + public IArrowArray GetChild(int index) + { + // TODO: Implement + throw new NotImplementedException(); + } + + public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor); + + } +} diff --git a/csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs b/csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs new file mode 100644 index 0000000000000..a85fa2dc9d949 --- /dev/null +++ b/csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Apache.Arrow +{ + public partial class ArrowBuffer + { + /// + /// Builds an Arrow buffer from primitive values. + /// + /// Primitive type + public class Builder + where T : struct + { + private readonly int _size; + private readonly MemoryPool _pool; + private Memory _memory; + private int _offset; + + public Builder(int initialCapacity = 8, MemoryPool pool = default) + { + if (initialCapacity <= 0) initialCapacity = 1; + if (pool == null) pool = DefaultMemoryPool.Instance.Value; + + _size = Unsafe.SizeOf(); + _pool = pool; + _memory = _pool.Allocate(initialCapacity * _size); + } + + public Builder Append(T value) + { + var span = GetSpan(); + + if (_offset + 1 >= span.Length) + { + // TODO: Consider a specifiable growth strategy + + _memory = _pool.Reallocate(_memory, (_memory.Length * 3) / 2); + } + + span[_offset++] = value; + return this; + } + + public Builder Set(int index, T value) + { + var span = GetSpan(); + span[index] = value; + return this; + } + + public Builder Clear() + { + var span = GetSpan(); + span.Fill(default); + return this; + } + + public ArrowBuffer Build() + { + return new ArrowBuffer(_memory, _offset); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private Span GetSpan() => MemoryMarshal.Cast(_memory.Span); + } + } +} diff --git a/csharp/src/Apache.Arrow/ArrowBuffer.cs b/csharp/src/Apache.Arrow/ArrowBuffer.cs new file mode 100644 index 0000000000000..ec2c3cbe8c123 --- /dev/null +++ b/csharp/src/Apache.Arrow/ArrowBuffer.cs @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; +using System; +using System.Buffers; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Apache.Arrow +{ + public partial class ArrowBuffer: IEquatable + { + public ArrowBuffer(Memory data, int size) + { + Memory = data; + Size = size; + } + + /// + /// Allocates an Arrow buffer from a memory pool. + /// + /// Size of buffer (in bytes) to allocate. + /// Memory pool to use for allocation. If null, a default memory pool is used. + /// + public static ArrowBuffer Allocate(int size, MemoryPool memoryPool = null) + { + if (memoryPool == null) + memoryPool = DefaultMemoryPool.Instance.Value; + + var buffer = memoryPool.Allocate(size); + + return new ArrowBuffer(buffer, size); + } + + /// + /// Allocates an Arrow buffer the same length as the incoming data, then + /// copies the specified data to the arrow buffer. + /// + /// Data to copy into a new arrow buffer. + /// Memory pool to use for allocation. If null, a default memory pool is used. + /// + public static ArrowBuffer FromMemory(Memory data, MemoryPool memoryPool = default) + { + var buffer = Allocate(data.Length, memoryPool); + data.CopyTo(buffer.Memory); + return buffer; + } + + public async Task CopyToAsync(Stream stream, CancellationToken cancellationToken = default) + { + const float chunkSize = 8192f; + + // TODO: Is there a better copy mechanism to use here that does not involve allocating buffers and targets .NET Standard 1.3? + // NOTE: Consider specialization for .NET Core 2.1 + + var length = Convert.ToInt32(chunkSize); + var buffer = ArrayPool.Shared.Rent(length); + var count = Convert.ToInt32(Math.Ceiling(Memory.Length / chunkSize)); + var offset = 0; + + try + { + for (var i = 0; i < count; i++) + { + var n = Math.Min(length, Memory.Length); + var slice = Memory.Slice(offset, n); + + slice.CopyTo(buffer); + + await stream.WriteAsync(buffer, 0, n, cancellationToken); + + offset += n; + } + } + finally + { + if (buffer != null) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + public Memory Memory { get; } + + public bool IsEmpty => Memory.IsEmpty; + + public int Size { get; } + + public int Capacity => Memory.Length; + + public Span GetSpan(int offset) + where T : struct => + MemoryMarshal.Cast( + Memory.Span.Slice(offset)); + + public Span GetSpan(int offset, int length) + where T : struct => + MemoryMarshal.Cast( + Memory.Span.Slice(offset, length)); + + public Span GetSpan() + where T: struct => + MemoryMarshal.Cast(Memory.Span); + + public bool Equals(ArrowBuffer other) + { + var lhs = GetSpan(); + var rhs = other.GetSpan(); + return lhs.SequenceEqual(rhs); + } + } +} diff --git a/csharp/src/Apache.Arrow/BitUtility.cs b/csharp/src/Apache.Arrow/BitUtility.cs new file mode 100644 index 0000000000000..ea5a556162e53 --- /dev/null +++ b/csharp/src/Apache.Arrow/BitUtility.cs @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics; + +namespace Apache.Arrow +{ + public static class BitUtility + { + private static readonly byte[] PopcountTable = new byte[] + { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, + }; + + private static readonly byte[] BitMask = new byte[] + { + 1, 2, 4, 8, 16, 32, 64, 128 + }; + + public static bool GetBit(byte data, int index) => + ((data >> index) & 1) != 0; + + public static bool GetBit(ReadOnlySpan data, int index) => + (data[index / 8] & BitMask[index % 8]) != 0; + + public static void ClearBit(Span data, int index) + { + data[index / 8] &= (byte) ~BitMask[index % 8]; + } + + public static void SetBit(Span data, int index) + { + data[index / 8] |= BitMask[index % 8]; + } + + /// + /// Counts the number of set bits in a span of bytes starting + /// at a specific bit offset. + /// + /// Span to count bits + /// Bit offset to start counting from + /// Count of set (one) bits/returns> + public static int CountBits(ReadOnlySpan data, int offset) + { + var start = (offset / 8); + var startBit = offset % 8; + + if (startBit < 0) return 0; + if (startBit == 0) return CountBits(data); + + var count = 0; + + count += CountBits(data.Slice(start + 1)); + + for (var i = startBit; i < 8; i++) + { + if (GetBit(data.Slice(start, 1), i)) + count++; + } + + return count; + } + + /// + /// Counts the number of set bits in a span of bytes. + /// + /// Span to count bits + /// Count of set (one) bits. + public static int CountBits(ReadOnlySpan data) + { + var count = 0; + foreach (var t in data) + count += PopcountTable[t]; + return count; + } + + /// + /// Rounds an integer to the nearest multiple of 64. + /// + /// Integer to round. + /// Integer rounded to the nearest multiple of 64. + public static int RoundUpToMultipleOf64(int n) => + RoundUpToMultiplePowerOfTwo(n, 64); + + /// + /// Rounds an integer up to the nearest multiple of factor, where + /// factor must be a power of two. + /// + /// This function does not throw when the factor is not a power of two. + /// + /// Integer to round up. + /// Power of two factor to round up to. + /// Integer rounded up to the nearest power of two. + public static int RoundUpToMultiplePowerOfTwo(int n, int factor) + { + // Assert that factor is a power of two. + Debug.Assert(factor > 0 && (factor & (factor - 1)) == 0); + return (n + (factor - 1)) & ~(factor - 1); + } + + } +} diff --git a/csharp/src/Apache.Arrow/Bitmap.cs b/csharp/src/Apache.Arrow/Bitmap.cs new file mode 100644 index 0000000000000..257438b323c7d --- /dev/null +++ b/csharp/src/Apache.Arrow/Bitmap.cs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Memory; + +namespace Apache.Arrow +{ + public struct Bitmap + { + public ArrowBuffer Buffer { get; } + + public int Length => Buffer.Size; + + public Bitmap(ArrowBuffer buffer) + { + Buffer = buffer; + } + + public static implicit operator Bitmap(ArrowBuffer buffer) + { + return new Bitmap(buffer); + } + + public static implicit operator ArrowBuffer(Bitmap bitmap) + { + return bitmap.Buffer; + } + + public static Bitmap Allocate(int bitCount, MemoryPool memoryPool = default) + { + var size = bitCount / 8 + (bitCount % 8 > 0 ? 1 : 0); + var remainder = size % 64; + var len = (remainder == 0) ? size : size + 64 - remainder; + + // Allocate buffer from memory pool and enable all bits + + var buffer = ArrowBuffer.Allocate(len, memoryPool); + var span = buffer.GetSpan(); + + span.Fill(0xff); + + return new Bitmap(buffer); + } + + public void Clear(int index) + { + BitUtility.ClearBit( + Buffer.GetSpan(), index); + } + + public void Set(int index) + { + BitUtility.SetBit( + Buffer.GetSpan(), index); + } + + public bool IsSet(int index) + { + return BitUtility.GetBit( + Buffer.GetSpan(), index); + } + } +} diff --git a/csharp/src/Apache.Arrow/Extensions/ArrayDataExtensions.cs b/csharp/src/Apache.Arrow/Extensions/ArrayDataExtensions.cs new file mode 100644 index 0000000000000..3cc4c139ecb34 --- /dev/null +++ b/csharp/src/Apache.Arrow/Extensions/ArrayDataExtensions.cs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; + +namespace Apache.Arrow +{ + internal static class ArrayDataExtensions + { + public static void EnsureBufferCount(this ArrayData data, int count) + { + if (data.Buffers.Length != count) + { + // TODO: Use localizable string resource + throw new ArgumentException( + $"Buffer count <{data.Buffers.Length}> must be at least <{count}>", + nameof(data.Buffers.Length)); + } + } + + public static void EnsureDataType(this ArrayData data, params ArrowTypeId[] ids) + { + var valid = true; + + foreach (var id in ids) + { + if (data.DataType.TypeId != id) + valid = false; + } + + if (!valid) + { + // TODO: Use localizable string resource + throw new ArgumentException( + $"Specified array type <{data.DataType.TypeId}> does not match expected type(s) <{string.Join(",", ids)}>", + nameof(data.DataType.TypeId)); + } + } + } +} diff --git a/csharp/src/Apache.Arrow/Extensions/ArrayPoolExtensions.cs b/csharp/src/Apache.Arrow/Extensions/ArrayPoolExtensions.cs new file mode 100644 index 0000000000000..e65a3ef86f425 --- /dev/null +++ b/csharp/src/Apache.Arrow/Extensions/ArrayPoolExtensions.cs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +namespace Apache.Arrow +{ + internal static class ArrayPoolExtensions + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void RentReturn(this ArrayPool pool, int length, Action action) + { + byte[] array = null; + + try + { + array = pool.Rent(length); + action(array); + } + finally + { + if (array != null) + { + pool.Return(array); + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Task RentReturnAsync(this ArrayPool pool, int length, Func action) + { + byte[] array = null; + + try + { + array = pool.Rent(length); + return action(array); + } + finally + { + if (array != null) + { + pool.Return(array); + } + } + } + } +} diff --git a/csharp/src/Apache.Arrow/Extensions/ArrowTypeExtensions.cs b/csharp/src/Apache.Arrow/Extensions/ArrowTypeExtensions.cs new file mode 100644 index 0000000000000..14e2b90a3c8a0 --- /dev/null +++ b/csharp/src/Apache.Arrow/Extensions/ArrowTypeExtensions.cs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public static class ArrowTypeExtensions + { + private static readonly ISet IntegralTypes = + new HashSet(new[] + { + ArrowTypeId.Int8, ArrowTypeId.Int16, ArrowTypeId.Int32, ArrowTypeId.Int64, + ArrowTypeId.UInt8, ArrowTypeId.UInt16, ArrowTypeId.UInt32, ArrowTypeId.UInt64, + }); + + private static readonly ISet FloatingPointTypes = + new HashSet(new[] + { + ArrowTypeId.HalfFloat, ArrowTypeId.Float, ArrowTypeId.Double + }); + + public static bool IsIntegral(this IArrowType type) + => IntegralTypes.Contains(type.TypeId); + + public static bool IsFloatingPoint(this IArrowType type) + => FloatingPointTypes.Contains(type.TypeId); + } +} diff --git a/csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs b/csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs new file mode 100644 index 0000000000000..d2a70bca9e4ec --- /dev/null +++ b/csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace Apache.Arrow +{ + internal static class FlatbufExtensions + { + public static bool IsFixedPrimitive(this Flatbuf.Type t) + { + if (t == Flatbuf.Type.Utf8 || t == Flatbuf.Type.Binary) + return false; + return true; + } + + public static bool IsFixedPrimitive(this Types.IArrowType t) + { + return t.TypeId.IsFixedPrimitive(); + } + + public static bool IsFixedPrimitive(this Types.ArrowTypeId t) + { + if (t == Types.ArrowTypeId.String || t == Types.ArrowTypeId.Binary) + return false; + return true; + } + + public static Types.IntervalUnit ToArrow(this Flatbuf.IntervalUnit unit) + { + switch (unit) + { + case Flatbuf.IntervalUnit.DAY_TIME: + return Types.IntervalUnit.DayTime; + case Flatbuf.IntervalUnit.YEAR_MONTH: + return Types.IntervalUnit.YearMonth; + default: + throw new ArgumentException($"Unexpected Flatbuf IntervalUnit", nameof(unit)); + } + } + + public static Types.DateUnit ToArrow(this Flatbuf.DateUnit unit) + { + switch (unit) + { + case Flatbuf.DateUnit.DAY: + return Types.DateUnit.Day; + case Flatbuf.DateUnit.MILLISECOND: + return Types.DateUnit.Milliseconds; + default: + throw new ArgumentException($"Unexpected Flatbuf IntervalUnit", nameof(unit)); + } + } + + public static Types.TimeUnit ToArrow(this Flatbuf.TimeUnit unit) + { + switch (unit) + { + case Flatbuf.TimeUnit.MICROSECOND: + return Types.TimeUnit.Microsecond; + case Flatbuf.TimeUnit.MILLISECOND: + return Types.TimeUnit.Millisecond; + case Flatbuf.TimeUnit.NANOSECOND: + return Types.TimeUnit.Nanosecond; + case Flatbuf.TimeUnit.SECOND: + return Types.TimeUnit.Second; + default: + throw new ArgumentException($"Unexpected Flatbuf TimeUnit", nameof(unit)); + } + } + } +} + diff --git a/csharp/src/Apache.Arrow/Field.Builder.cs b/csharp/src/Apache.Arrow/Field.Builder.cs new file mode 100644 index 0000000000000..fede4ea73f8f3 --- /dev/null +++ b/csharp/src/Apache.Arrow/Field.Builder.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Types; +using System; +using System.Collections.Generic; + +namespace Apache.Arrow +{ + public partial class Field + { + public class Builder + { + private readonly Dictionary _metadata; + private string _name; + private IArrowType _type; + private bool _nullable; + + public Builder() + { + _metadata = new Dictionary(); + _name = string.Empty; + _type = NullType.Default; + _nullable = true; + } + + public Builder Name(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentNullException(nameof(value)); + } + + _name = value; + return this; + } + + public Builder DataType(IArrowType type) + { + _type = type ?? NullType.Default; + return this; + } + + public Builder Nullable(bool value) + { + _nullable = value; + return this; + } + + public Builder Metadata(string key, string value) + { + if (string.IsNullOrWhiteSpace(key)) + { + throw new ArgumentNullException(nameof(key)); + } + + _metadata[key] = value; + return this; + } + + public Field Build() + { + return new Field(_name, _type, _nullable, _metadata); + } + } + } +} diff --git a/csharp/src/Apache.Arrow/Field.cs b/csharp/src/Apache.Arrow/Field.cs new file mode 100644 index 0000000000000..33aefb7030f8f --- /dev/null +++ b/csharp/src/Apache.Arrow/Field.cs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Apache.Arrow.Types; + +namespace Apache.Arrow +{ + public partial class Field + { + public IArrowType DataType { get; } + + public string Name { get; } + + public bool IsNullable { get; } + + public bool HasMetadata => Metadata?.Count > 0; + + public IReadOnlyDictionary Metadata { get; } + + public Field(string name, IArrowType dataType, bool nullable, + IEnumerable> metadata = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentNullException(nameof(name)); + } + + Name = name; + DataType = dataType ?? NullType.Default; + IsNullable = nullable; + Metadata = metadata?.ToDictionary(kv => kv.Key, kv => kv.Value); + } + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Block.cs b/csharp/src/Apache.Arrow/Flatbuf/Block.cs new file mode 100644 index 0000000000000..ec97fd19bec9f --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Block.cs @@ -0,0 +1,37 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +public struct Block : IFlatbufferObject +{ + private Struct __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Block __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// Index to the start of the RecordBlock (note this is past the Message header) + public long Offset { get { return __p.bb.GetLong(__p.bb_pos + 0); } } + /// Length of the metadata + public int MetaDataLength { get { return __p.bb.GetInt(__p.bb_pos + 8); } } + /// Length of the data (this is aligned so there can be a gap between this and + /// the metatdata). + public long BodyLength { get { return __p.bb.GetLong(__p.bb_pos + 16); } } + + public static Offset CreateBlock(FlatBufferBuilder builder, long Offset, int MetaDataLength, long BodyLength) { + builder.Prep(8, 24); + builder.PutLong(BodyLength); + builder.Pad(4); + builder.PutInt(MetaDataLength); + builder.PutLong(Offset); + return new Offset(builder.Offset); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs b/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs new file mode 100644 index 0000000000000..25506f8f578ea --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Buffer.cs @@ -0,0 +1,36 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// A Buffer represents a single contiguous memory segment +public struct Buffer : IFlatbufferObject +{ + private Struct __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Buffer __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// The relative offset into the shared memory page where the bytes for this + /// buffer starts + public long Offset { get { return __p.bb.GetLong(__p.bb_pos + 0); } } + /// The absolute length (in bytes) of the memory buffer. The memory is found + /// from offset (inclusive) to offset + length (non-inclusive). + public long Length { get { return __p.bb.GetLong(__p.bb_pos + 8); } } + + public static Offset CreateBuffer(FlatBufferBuilder builder, long Offset, long Length) { + builder.Prep(8, 16); + builder.PutLong(Length); + builder.PutLong(Offset); + return new Offset(builder.Offset); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs b/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs new file mode 100644 index 0000000000000..e16bf15992728 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs @@ -0,0 +1,54 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// For sending dictionary encoding information. Any Field can be +/// dictionary-encoded, but in this case none of its children may be +/// dictionary-encoded. +/// There is one vector / column per dictionary, but that vector / column +/// may be spread across multiple dictionary batches by using the isDelta +/// flag +public struct DictionaryBatch : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static DictionaryBatch GetRootAsDictionaryBatch(ByteBuffer _bb) { return GetRootAsDictionaryBatch(_bb, new DictionaryBatch()); } + public static DictionaryBatch GetRootAsDictionaryBatch(ByteBuffer _bb, DictionaryBatch obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public DictionaryBatch __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + public long Id { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } } + public RecordBatch? Data { get { int o = __p.__offset(6); return o != 0 ? (RecordBatch?)(new RecordBatch()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + /// If isDelta is true the values in the dictionary are to be appended to a + /// dictionary with the indicated id + public bool IsDelta { get { int o = __p.__offset(8); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } } + + public static Offset CreateDictionaryBatch(FlatBufferBuilder builder, + long id = 0, + Offset dataOffset = default(Offset), + bool isDelta = false) { + builder.StartObject(3); + DictionaryBatch.AddId(builder, id); + DictionaryBatch.AddData(builder, dataOffset); + DictionaryBatch.AddIsDelta(builder, isDelta); + return DictionaryBatch.EndDictionaryBatch(builder); + } + + public static void StartDictionaryBatch(FlatBufferBuilder builder) { builder.StartObject(3); } + public static void AddId(FlatBufferBuilder builder, long id) { builder.AddLong(0, id, 0); } + public static void AddData(FlatBufferBuilder builder, Offset dataOffset) { builder.AddOffset(1, dataOffset.Value, 0); } + public static void AddIsDelta(FlatBufferBuilder builder, bool isDelta) { builder.AddBool(2, isDelta, false); } + public static Offset EndDictionaryBatch(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs b/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs new file mode 100644 index 0000000000000..282caf304eb93 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs @@ -0,0 +1,57 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// Dictionary encoding metadata +public struct DictionaryEncoding : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static DictionaryEncoding GetRootAsDictionaryEncoding(ByteBuffer _bb) { return GetRootAsDictionaryEncoding(_bb, new DictionaryEncoding()); } + public static DictionaryEncoding GetRootAsDictionaryEncoding(ByteBuffer _bb, DictionaryEncoding obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public DictionaryEncoding __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// The known dictionary id in the application where this data is used. In + /// the file or streaming formats, the dictionary ids are found in the + /// DictionaryBatch messages + public long Id { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } } + /// The dictionary indices are constrained to be positive integers. If this + /// field is null, the indices must be signed int32 + public Int? IndexType { get { int o = __p.__offset(6); return o != 0 ? (Int?)(new Int()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + /// By default, dictionaries are not ordered, or the order does not have + /// semantic meaning. In some statistical, applications, dictionary-encoding + /// is used to represent ordered categorical data, and we provide a way to + /// preserve that metadata here + public bool IsOrdered { get { int o = __p.__offset(8); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } } + + public static Offset CreateDictionaryEncoding(FlatBufferBuilder builder, + long id = 0, + Offset indexTypeOffset = default(Offset), + bool isOrdered = false) { + builder.StartObject(3); + DictionaryEncoding.AddId(builder, id); + DictionaryEncoding.AddIndexType(builder, indexTypeOffset); + DictionaryEncoding.AddIsOrdered(builder, isOrdered); + return DictionaryEncoding.EndDictionaryEncoding(builder); + } + + public static void StartDictionaryEncoding(FlatBufferBuilder builder) { builder.StartObject(3); } + public static void AddId(FlatBufferBuilder builder, long id) { builder.AddLong(0, id, 0); } + public static void AddIndexType(FlatBufferBuilder builder, Offset indexTypeOffset) { builder.AddOffset(1, indexTypeOffset.Value, 0); } + public static void AddIsOrdered(FlatBufferBuilder builder, bool isOrdered) { builder.AddBool(2, isOrdered, false); } + public static Offset EndDictionaryEncoding(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs new file mode 100644 index 0000000000000..66c0740204f84 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs @@ -0,0 +1,15 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +public enum DateUnit : short +{ + DAY = 0, + MILLISECOND = 1, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs new file mode 100644 index 0000000000000..0609f674c6bd4 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs @@ -0,0 +1,17 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// ---------------------------------------------------------------------- +/// Endianness of the platform producing the data +public enum Endianness : short +{ + Little = 0, + Big = 1, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs new file mode 100644 index 0000000000000..9134b704072ff --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs @@ -0,0 +1,15 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +public enum IntervalUnit : short +{ + YEAR_MONTH = 0, + DAY_TIME = 1, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs new file mode 100644 index 0000000000000..7334de6489650 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs @@ -0,0 +1,26 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// ---------------------------------------------------------------------- +/// The root Message type +/// This union enables us to easily send different message types without +/// redundant storage, and in the future we can easily add new message types. +/// +/// Arrow implementations do not need to implement all of the message types, +/// which may include experimental metadata types. For maximum compatibility, +/// it is best to send data using RecordBatch +public enum MessageHeader : byte +{ + NONE = 0, + Schema = 1, + DictionaryBatch = 2, + RecordBatch = 3, + Tensor = 4, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs new file mode 100644 index 0000000000000..3b97c2f623aca --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs @@ -0,0 +1,21 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +public enum MetadataVersion : short +{ + /// 0.1.0 + V1 = 0, + /// 0.2.0 + V2 = 1, + /// 0.3.0 -> 0.7.1 + V3 = 2, + /// >= 0.8.0 + V4 = 3, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs new file mode 100644 index 0000000000000..8c8552c405d23 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs @@ -0,0 +1,16 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +public enum Precision : short +{ + HALF = 0, + SINGLE = 1, + DOUBLE = 2, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs new file mode 100644 index 0000000000000..8a96de0256644 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs @@ -0,0 +1,17 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +public enum TimeUnit : short +{ + SECOND = 0, + MILLISECOND = 1, + MICROSECOND = 2, + NANOSECOND = 3, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs new file mode 100644 index 0000000000000..dce9ac8dd0fe6 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs @@ -0,0 +1,34 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +/// ---------------------------------------------------------------------- +/// Top-level Type value, enabling extensible type-specific metadata. We can +/// add new logical types to Type without breaking backwards compatibility +public enum Type : byte +{ + NONE = 0, + Null = 1, + Int = 2, + FloatingPoint = 3, + Binary = 4, + Utf8 = 5, + Bool = 6, + Decimal = 7, + Date = 8, + Time = 9, + Timestamp = 10, + Interval = 11, + List = 12, + Struct_ = 13, + Union = 14, + FixedSizeBinary = 15, + FixedSizeList = 16, + Map = 17, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs b/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs new file mode 100644 index 0000000000000..bb6ddd3627ff4 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs @@ -0,0 +1,15 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +public enum UnionMode : short +{ + Sparse = 0, + Dense = 1, +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Field.cs b/csharp/src/Apache.Arrow/Flatbuf/Field.cs new file mode 100644 index 0000000000000..a75f2b563fa5e --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Field.cs @@ -0,0 +1,76 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// A field represents a named column in a record / row batch or child of a +/// nested type. +/// +/// - children is only for nested Arrow arrays +/// - For primitive types, children will have length 0 +/// - nullable should default to true in general +public struct Field : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static Field GetRootAsField(ByteBuffer _bb) { return GetRootAsField(_bb, new Field()); } + public static Field GetRootAsField(ByteBuffer _bb, Field obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Field __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + public string Name { get { int o = __p.__offset(4); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } } + public ArraySegment? GetNameBytes() { return __p.__vector_as_arraysegment(4); } + public bool Nullable { get { int o = __p.__offset(6); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } } + public Type TypeType { get { int o = __p.__offset(8); return o != 0 ? (Type)__p.bb.Get(o + __p.bb_pos) : Flatbuf.Type.NONE; } } + public TTable? Type() where TTable : struct, IFlatbufferObject { int o = __p.__offset(10); return o != 0 ? (TTable?)__p.__union(o) : null; } + public DictionaryEncoding? Dictionary { get { int o = __p.__offset(12); return o != 0 ? (DictionaryEncoding?)(new DictionaryEncoding()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + public Field? Children(int j) { int o = __p.__offset(14); return o != 0 ? (Field?)(new Field()).__assign(__p.__indirect(__p.__vector(o) + j * 4), __p.bb) : null; } + public int ChildrenLength { get { int o = __p.__offset(14); return o != 0 ? __p.__vector_len(o) : 0; } } + public KeyValue? CustomMetadata(int j) { int o = __p.__offset(16); return o != 0 ? (KeyValue?)(new KeyValue()).__assign(__p.__indirect(__p.__vector(o) + j * 4), __p.bb) : null; } + public int CustomMetadataLength { get { int o = __p.__offset(16); return o != 0 ? __p.__vector_len(o) : 0; } } + + public static Offset CreateField(FlatBufferBuilder builder, + StringOffset nameOffset = default(StringOffset), + bool nullable = false, + Type type_type = Flatbuf.Type.NONE, + int typeOffset = 0, + Offset dictionaryOffset = default(Offset), + VectorOffset childrenOffset = default(VectorOffset), + VectorOffset custom_metadataOffset = default(VectorOffset)) { + builder.StartObject(7); + Field.AddCustomMetadata(builder, custom_metadataOffset); + Field.AddChildren(builder, childrenOffset); + Field.AddDictionary(builder, dictionaryOffset); + Field.AddType(builder, typeOffset); + Field.AddName(builder, nameOffset); + Field.AddTypeType(builder, type_type); + Field.AddNullable(builder, nullable); + return Field.EndField(builder); + } + + public static void StartField(FlatBufferBuilder builder) { builder.StartObject(7); } + public static void AddName(FlatBufferBuilder builder, StringOffset nameOffset) { builder.AddOffset(0, nameOffset.Value, 0); } + public static void AddNullable(FlatBufferBuilder builder, bool nullable) { builder.AddBool(1, nullable, false); } + public static void AddTypeType(FlatBufferBuilder builder, Type typeType) { builder.AddByte(2, (byte)typeType, 0); } + public static void AddType(FlatBufferBuilder builder, int typeOffset) { builder.AddOffset(3, typeOffset, 0); } + public static void AddDictionary(FlatBufferBuilder builder, Offset dictionaryOffset) { builder.AddOffset(4, dictionaryOffset.Value, 0); } + public static void AddChildren(FlatBufferBuilder builder, VectorOffset childrenOffset) { builder.AddOffset(5, childrenOffset.Value, 0); } + public static VectorOffset CreateChildrenVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static void StartChildrenVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } + public static void AddCustomMetadata(FlatBufferBuilder builder, VectorOffset customMetadataOffset) { builder.AddOffset(6, customMetadataOffset.Value, 0); } + public static VectorOffset CreateCustomMetadataVector(FlatBufferBuilder builder, Offset[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); } + public static void StartCustomMetadataVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); } + public static Offset EndField(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs b/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs new file mode 100644 index 0000000000000..53b01b8f9a604 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs @@ -0,0 +1,44 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// Data structures for describing a table row batch (a collection of +/// equal-length Arrow arrays) +/// Metadata about a field at some level of a nested type tree (but not +/// its children). +/// +/// For example, a List with values [[1, 2, 3], null, [4], [5, 6], null] +/// would have {length: 5, null_count: 2} for its List node, and {length: 6, +/// null_count: 0} for its Int16 node, as separate FieldNode structs +public struct FieldNode : IFlatbufferObject +{ + private Struct __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public FieldNode __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// The number of value slots in the Arrow array at this level of a nested + /// tree + public long Length { get { return __p.bb.GetLong(__p.bb_pos + 0); } } + /// The number of observed nulls. Fields with null_count == 0 may choose not + /// to write their physical validity bitmap out as a materialized buffer, + /// instead setting the length of the bitmap buffer to 0. + public long NullCount { get { return __p.bb.GetLong(__p.bb_pos + 8); } } + + public static Offset CreateFieldNode(FlatBufferBuilder builder, long Length, long NullCount) { + builder.Prep(8, 16); + builder.PutLong(NullCount); + builder.PutLong(Length); + return new Offset(builder.Offset); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs new file mode 100644 index 0000000000000..a022af7e0c1cf --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs @@ -0,0 +1,39 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +public struct FixedSizeBinary : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static FixedSizeBinary GetRootAsFixedSizeBinary(ByteBuffer _bb) { return GetRootAsFixedSizeBinary(_bb, new FixedSizeBinary()); } + public static FixedSizeBinary GetRootAsFixedSizeBinary(ByteBuffer _bb, FixedSizeBinary obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public FixedSizeBinary __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// Number of bytes per value + public int ByteWidth { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetInt(o + __p.bb_pos) : (int)0; } } + + public static Offset CreateFixedSizeBinary(FlatBufferBuilder builder, + int byteWidth = 0) { + builder.StartObject(1); + FixedSizeBinary.AddByteWidth(builder, byteWidth); + return FixedSizeBinary.EndFixedSizeBinary(builder); + } + + public static void StartFixedSizeBinary(FlatBufferBuilder builder) { builder.StartObject(1); } + public static void AddByteWidth(FlatBufferBuilder builder, int byteWidth) { builder.AddInt(0, byteWidth, 0); } + public static Offset EndFixedSizeBinary(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs new file mode 100644 index 0000000000000..12a23d980d8e6 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs @@ -0,0 +1,39 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +public struct FixedSizeList : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static FixedSizeList GetRootAsFixedSizeList(ByteBuffer _bb) { return GetRootAsFixedSizeList(_bb, new FixedSizeList()); } + public static FixedSizeList GetRootAsFixedSizeList(ByteBuffer _bb, FixedSizeList obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public FixedSizeList __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + /// Number of list items per value + public int ListSize { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetInt(o + __p.bb_pos) : (int)0; } } + + public static Offset CreateFixedSizeList(FlatBufferBuilder builder, + int listSize = 0) { + builder.StartObject(1); + FixedSizeList.AddListSize(builder, listSize); + return FixedSizeList.EndFixedSizeList(builder); + } + + public static void StartFixedSizeList(FlatBufferBuilder builder) { builder.StartObject(1); } + public static void AddListSize(FlatBufferBuilder builder, int listSize) { builder.AddInt(0, listSize, 0); } + public static Offset EndFixedSizeList(FlatBufferBuilder builder) { + int o = builder.EndObject(); + return new Offset(o); + } +}; + + +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs new file mode 100644 index 0000000000000..307a98fadca75 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs @@ -0,0 +1,633 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// There are 2 #defines that have an impact on performance of this ByteBuffer implementation +// +// UNSAFE_BYTEBUFFER +// This will use unsafe code to manipulate the underlying byte array. This +// can yield a reasonable performance increase. +// +// BYTEBUFFER_NO_BOUNDS_CHECK +// This will disable the bounds check asserts to the byte array. This can +// yield a small performance gain in normal code.. +// +// Using UNSAFE_BYTEBUFFER and BYTEBUFFER_NO_BOUNDS_CHECK together can yield a +// performance gain of ~15% for some operations, however doing so is potentially +// dangerous. Do so at your own risk! +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace FlatBuffers +{ + /// + /// Class to mimic Java's ByteBuffer which is used heavily in Flatbuffers. + /// + public class ByteBuffer + { + protected byte[] _buffer; + private int _pos; // Must track start of the buffer. + + public int Length { get { return _buffer.Length; } } + + public ByteBuffer(int size) : this(new byte[size]) { } + + public ByteBuffer(byte[] buffer) : this(buffer, 0) { } + + public ByteBuffer(byte[] buffer, int pos) + { + _buffer = buffer; + _pos = pos; + } + + public int Position { + get { return _pos; } + set { _pos = value; } + } + + public void Reset() + { + _pos = 0; + } + + // Create a new ByteBuffer on the same underlying data. + // The new ByteBuffer's position will be same as this buffer's. + public ByteBuffer Duplicate() + { + return new ByteBuffer(_buffer, Position); + } + + // Increases the size of the ByteBuffer, and copies the old data towards + // the end of the new buffer. + public void GrowFront(int newSize) + { + if ((Length & 0xC0000000) != 0) + throw new Exception( + "ByteBuffer: cannot grow buffer beyond 2 gigabytes."); + + if (newSize < Length) + throw new Exception("ByteBuffer: cannot truncate buffer."); + + byte[] newBuffer = new byte[newSize]; + Buffer.BlockCopy(_buffer, 0, newBuffer, newSize - Length, + Length); + _buffer = newBuffer; + } + + public byte[] ToArray(int pos, int len) + { + return ToArray(pos, len); + } + + /// + /// A lookup of type sizes. Used instead of Marshal.SizeOf() which has additional + /// overhead, but also is compatible with generic functions for simplified code. + /// + private static Dictionary genericSizes = new Dictionary() + { + { typeof(bool), sizeof(bool) }, + { typeof(float), sizeof(float) }, + { typeof(double), sizeof(double) }, + { typeof(sbyte), sizeof(sbyte) }, + { typeof(byte), sizeof(byte) }, + { typeof(short), sizeof(short) }, + { typeof(ushort), sizeof(ushort) }, + { typeof(int), sizeof(int) }, + { typeof(uint), sizeof(uint) }, + { typeof(ulong), sizeof(ulong) }, + { typeof(long), sizeof(long) }, + }; + + /// + /// Get the wire-size (in bytes) of a type supported by flatbuffers. + /// + /// The type to get the wire size of + /// + public static int SizeOf() + { + return genericSizes[typeof(T)]; + } + + /// + /// Checks if the Type provided is supported as scalar value + /// + /// The Type to check + /// True if the type is a scalar type that is supported, falsed otherwise + public static bool IsSupportedType() + { + return genericSizes.ContainsKey(typeof(T)); + } + + /// + /// Get the wire-size (in bytes) of an typed array + /// + /// The type of the array + /// The array to get the size of + /// The number of bytes the array takes on wire + public static int ArraySize(T[] x) + { + return SizeOf() * x.Length; + } + + // Get a portion of the buffer casted into an array of type T, given + // the buffer position and length. + public T[] ToArray(int pos, int len) + where T: struct + { + AssertOffsetAndLength(pos, len); + T[] arr = new T[len]; + Buffer.BlockCopy(_buffer, pos, arr, 0, ArraySize(arr)); + return arr; + } + + public byte[] ToSizedArray() + { + return ToArray(Position, Length - Position); + } + + public byte[] ToFullArray() + { + return ToArray(0, Length); + } + + public ArraySegment ToArraySegment(int pos, int len) + { + return new ArraySegment(_buffer, pos, len); + } + + public MemoryStream ToMemoryStream(int pos, int len) + { + return new MemoryStream(_buffer, pos, len); + } + +#if !UNSAFE_BYTEBUFFER + // Pre-allocated helper arrays for convertion. + private float[] floathelper = new[] { 0.0f }; + private int[] inthelper = new[] { 0 }; + private double[] doublehelper = new[] { 0.0 }; + private ulong[] ulonghelper = new[] { 0UL }; +#endif // !UNSAFE_BYTEBUFFER + + // Helper functions for the unsafe version. + static public ushort ReverseBytes(ushort input) + { + return (ushort)(((input & 0x00FFU) << 8) | + ((input & 0xFF00U) >> 8)); + } + static public uint ReverseBytes(uint input) + { + return ((input & 0x000000FFU) << 24) | + ((input & 0x0000FF00U) << 8) | + ((input & 0x00FF0000U) >> 8) | + ((input & 0xFF000000U) >> 24); + } + static public ulong ReverseBytes(ulong input) + { + return (((input & 0x00000000000000FFUL) << 56) | + ((input & 0x000000000000FF00UL) << 40) | + ((input & 0x0000000000FF0000UL) << 24) | + ((input & 0x00000000FF000000UL) << 8) | + ((input & 0x000000FF00000000UL) >> 8) | + ((input & 0x0000FF0000000000UL) >> 24) | + ((input & 0x00FF000000000000UL) >> 40) | + ((input & 0xFF00000000000000UL) >> 56)); + } + +#if !UNSAFE_BYTEBUFFER + // Helper functions for the safe (but slower) version. + protected void WriteLittleEndian(int offset, int count, ulong data) + { + if (BitConverter.IsLittleEndian) + { + for (int i = 0; i < count; i++) + { + _buffer[offset + i] = (byte)(data >> i * 8); + } + } + else + { + for (int i = 0; i < count; i++) + { + _buffer[offset + count - 1 - i] = (byte)(data >> i * 8); + } + } + } + + protected ulong ReadLittleEndian(int offset, int count) + { + AssertOffsetAndLength(offset, count); + ulong r = 0; + if (BitConverter.IsLittleEndian) + { + for (int i = 0; i < count; i++) + { + r |= (ulong)_buffer[offset + i] << i * 8; + } + } + else + { + for (int i = 0; i < count; i++) + { + r |= (ulong)_buffer[offset + count - 1 - i] << i * 8; + } + } + return r; + } +#endif // !UNSAFE_BYTEBUFFER + + private void AssertOffsetAndLength(int offset, int length) + { + #if !BYTEBUFFER_NO_BOUNDS_CHECK + if (offset < 0 || + offset > _buffer.Length - length) + throw new ArgumentOutOfRangeException(); + #endif + } + + public void PutSbyte(int offset, sbyte value) + { + AssertOffsetAndLength(offset, sizeof(sbyte)); + _buffer[offset] = (byte)value; + } + + public void PutByte(int offset, byte value) + { + AssertOffsetAndLength(offset, sizeof(byte)); + _buffer[offset] = value; + } + + public void PutByte(int offset, byte value, int count) + { + AssertOffsetAndLength(offset, sizeof(byte) * count); + for (var i = 0; i < count; ++i) + _buffer[offset + i] = value; + } + + // this method exists in order to conform with Java ByteBuffer standards + public void Put(int offset, byte value) + { + PutByte(offset, value); + } + + public void PutStringUTF8(int offset, string value) + { + AssertOffsetAndLength(offset, value.Length); + Encoding.UTF8.GetBytes(value, 0, value.Length, + _buffer, offset); + } + +#if UNSAFE_BYTEBUFFER + // Unsafe but more efficient versions of Put*. + public void PutShort(int offset, short value) + { + PutUshort(offset, (ushort)value); + } + + public unsafe void PutUshort(int offset, ushort value) + { + AssertOffsetAndLength(offset, sizeof(ushort)); + fixed (byte* ptr = _buffer) + { + *(ushort*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); + } + } + + public void PutInt(int offset, int value) + { + PutUint(offset, (uint)value); + } + + public unsafe void PutUint(int offset, uint value) + { + AssertOffsetAndLength(offset, sizeof(uint)); + fixed (byte* ptr = _buffer) + { + *(uint*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); + } + } + + public unsafe void PutLong(int offset, long value) + { + PutUlong(offset, (ulong)value); + } + + public unsafe void PutUlong(int offset, ulong value) + { + AssertOffsetAndLength(offset, sizeof(ulong)); + fixed (byte* ptr = _buffer) + { + *(ulong*)(ptr + offset) = BitConverter.IsLittleEndian + ? value + : ReverseBytes(value); + } + } + + public unsafe void PutFloat(int offset, float value) + { + AssertOffsetAndLength(offset, sizeof(float)); + fixed (byte* ptr = _buffer) + { + if (BitConverter.IsLittleEndian) + { + *(float*)(ptr + offset) = value; + } + else + { + *(uint*)(ptr + offset) = ReverseBytes(*(uint*)(&value)); + } + } + } + + public unsafe void PutDouble(int offset, double value) + { + AssertOffsetAndLength(offset, sizeof(double)); + fixed (byte* ptr = _buffer) + { + if (BitConverter.IsLittleEndian) + { + *(double*)(ptr + offset) = value; + + } + else + { + *(ulong*)(ptr + offset) = ReverseBytes(*(ulong*)(ptr + offset)); + } + } + } +#else // !UNSAFE_BYTEBUFFER + // Slower versions of Put* for when unsafe code is not allowed. + public void PutShort(int offset, short value) + { + AssertOffsetAndLength(offset, sizeof(short)); + WriteLittleEndian(offset, sizeof(short), (ulong)value); + } + + public void PutUshort(int offset, ushort value) + { + AssertOffsetAndLength(offset, sizeof(ushort)); + WriteLittleEndian(offset, sizeof(ushort), (ulong)value); + } + + public void PutInt(int offset, int value) + { + AssertOffsetAndLength(offset, sizeof(int)); + WriteLittleEndian(offset, sizeof(int), (ulong)value); + } + + public void PutUint(int offset, uint value) + { + AssertOffsetAndLength(offset, sizeof(uint)); + WriteLittleEndian(offset, sizeof(uint), (ulong)value); + } + + public void PutLong(int offset, long value) + { + AssertOffsetAndLength(offset, sizeof(long)); + WriteLittleEndian(offset, sizeof(long), (ulong)value); + } + + public void PutUlong(int offset, ulong value) + { + AssertOffsetAndLength(offset, sizeof(ulong)); + WriteLittleEndian(offset, sizeof(ulong), value); + } + + public void PutFloat(int offset, float value) + { + AssertOffsetAndLength(offset, sizeof(float)); + floathelper[0] = value; + Buffer.BlockCopy(floathelper, 0, inthelper, 0, sizeof(float)); + WriteLittleEndian(offset, sizeof(float), (ulong)inthelper[0]); + } + + public void PutDouble(int offset, double value) + { + AssertOffsetAndLength(offset, sizeof(double)); + doublehelper[0] = value; + Buffer.BlockCopy(doublehelper, 0, ulonghelper, 0, sizeof(double)); + WriteLittleEndian(offset, sizeof(double), ulonghelper[0]); + } + + /// + /// Copies an array of type T into this buffer, ending at the given + /// offset into this buffer. The starting offset is calculated based on the length + /// of the array and is the value returned. + /// + /// The type of the input data (must be a struct) + /// The offset into this buffer where the copy will end + /// The array to copy data from + /// The 'start' location of this buffer now, after the copy completed + public int Put(int offset, T[] x) + where T : struct + { + if(x == null) + { + throw new ArgumentNullException("Cannot put a null array"); + } + + if(x.Length == 0) + { + throw new ArgumentException("Cannot put an empty array"); + } + + if(!IsSupportedType()) + { + throw new ArgumentException("Cannot put an array of type " + + typeof(T) + " into this buffer"); + } + + if (BitConverter.IsLittleEndian) + { + int numBytes = ByteBuffer.ArraySize(x); + offset -= numBytes; + AssertOffsetAndLength(offset, numBytes); + // if we are LE, just do a block copy + Buffer.BlockCopy(x, 0, _buffer, offset, numBytes); + } + else + { + throw new NotImplementedException("Big Endian Support not implemented yet " + + "for putting typed arrays"); + // if we are BE, we have to swap each element by itself + //for(int i = x.Length - 1; i >= 0; i--) + //{ + // todo: low priority, but need to genericize the Put() functions + //} + } + return offset; + } + + +#endif // UNSAFE_BYTEBUFFER + + public sbyte GetSbyte(int index) + { + AssertOffsetAndLength(index, sizeof(sbyte)); + return (sbyte)_buffer[index]; + } + + public byte Get(int index) + { + AssertOffsetAndLength(index, sizeof(byte)); + return _buffer[index]; + } + + public string GetStringUTF8(int startPos, int len) + { + return Encoding.UTF8.GetString(_buffer, startPos, len); + } + +#if UNSAFE_BYTEBUFFER + // Unsafe but more efficient versions of Get*. + public short GetShort(int offset) + { + return (short)GetUshort(offset); + } + + public unsafe ushort GetUshort(int offset) + { + AssertOffsetAndLength(offset, sizeof(ushort)); + fixed (byte* ptr = _buffer) + { + return BitConverter.IsLittleEndian + ? *(ushort*)(ptr + offset) + : ReverseBytes(*(ushort*)(ptr + offset)); + } + } + + public int GetInt(int offset) + { + return (int)GetUint(offset); + } + + public unsafe uint GetUint(int offset) + { + AssertOffsetAndLength(offset, sizeof(uint)); + fixed (byte* ptr = _buffer) + { + return BitConverter.IsLittleEndian + ? *(uint*)(ptr + offset) + : ReverseBytes(*(uint*)(ptr + offset)); + } + } + + public long GetLong(int offset) + { + return (long)GetUlong(offset); + } + + public unsafe ulong GetUlong(int offset) + { + AssertOffsetAndLength(offset, sizeof(ulong)); + fixed (byte* ptr = _buffer) + { + return BitConverter.IsLittleEndian + ? *(ulong*)(ptr + offset) + : ReverseBytes(*(ulong*)(ptr + offset)); + } + } + + public unsafe float GetFloat(int offset) + { + AssertOffsetAndLength(offset, sizeof(float)); + fixed (byte* ptr = _buffer) + { + if (BitConverter.IsLittleEndian) + { + return *(float*)(ptr + offset); + } + else + { + uint uvalue = ReverseBytes(*(uint*)(ptr + offset)); + return *(float*)(&uvalue); + } + } + } + + public unsafe double GetDouble(int offset) + { + AssertOffsetAndLength(offset, sizeof(double)); + fixed (byte* ptr = _buffer) + { + if (BitConverter.IsLittleEndian) + { + return *(double*)(ptr + offset); + } + else + { + ulong uvalue = ReverseBytes(*(ulong*)(ptr + offset)); + return *(double*)(&uvalue); + } + } + } +#else // !UNSAFE_BYTEBUFFER + // Slower versions of Get* for when unsafe code is not allowed. + public short GetShort(int index) + { + return (short)ReadLittleEndian(index, sizeof(short)); + } + + public ushort GetUshort(int index) + { + return (ushort)ReadLittleEndian(index, sizeof(ushort)); + } + + public int GetInt(int index) + { + return (int)ReadLittleEndian(index, sizeof(int)); + } + + public uint GetUint(int index) + { + return (uint)ReadLittleEndian(index, sizeof(uint)); + } + + public long GetLong(int index) + { + return (long)ReadLittleEndian(index, sizeof(long)); + } + + public ulong GetUlong(int index) + { + return ReadLittleEndian(index, sizeof(ulong)); + } + + public float GetFloat(int index) + { + int i = (int)ReadLittleEndian(index, sizeof(float)); + inthelper[0] = i; + Buffer.BlockCopy(inthelper, 0, floathelper, 0, sizeof(float)); + return floathelper[0]; + } + + public double GetDouble(int index) + { + ulong i = ReadLittleEndian(index, sizeof(double)); + // There's Int64BitsToDouble but it uses unsafe code internally. + ulonghelper[0] = i; + Buffer.BlockCopy(ulonghelper, 0, doublehelper, 0, sizeof(double)); + return doublehelper[0]; + } +#endif // UNSAFE_BYTEBUFFER + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs new file mode 100644 index 0000000000000..66e8266843ba6 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs @@ -0,0 +1,39 @@ +/* + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace FlatBuffers +{ + /// + /// Class that collects utility functions around `ByteBuffer`. + /// + public class ByteBufferUtil + { + // Extract the size prefix from a `ByteBuffer`. + public static int GetSizePrefix(ByteBuffer bb) { + return bb.GetInt(bb.Position); + } + + // Create a duplicate of a size-prefixed `ByteBuffer` that has its position + // advanced just past the size prefix. + public static ByteBuffer RemoveSizePrefix(ByteBuffer bb) { + ByteBuffer s = bb.Duplicate(); + s.Position += FlatBufferConstants.SizePrefixLength; + return s; + } + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs new file mode 100644 index 0000000000000..33bba969ddac6 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs @@ -0,0 +1,744 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +using System; +using System.Text; + +/// @file +/// @addtogroup flatbuffers_csharp_api +/// @{ + +namespace FlatBuffers +{ + /// + /// Responsible for building up and accessing a FlatBuffer formatted byte + /// array (via ByteBuffer). + /// + public class FlatBufferBuilder + { + private int _space; + private ByteBuffer _bb; + private int _minAlign = 1; + + // The vtable for the current table (if _vtableSize >= 0) + private int[] _vtable = new int[16]; + // The size of the vtable. -1 indicates no vtable + private int _vtableSize = -1; + // Starting offset of the current struct/table. + private int _objectStart; + // List of offsets of all vtables. + private int[] _vtables = new int[16]; + // Number of entries in `vtables` in use. + private int _numVtables = 0; + // For the current vector being built. + private int _vectorNumElems = 0; + + /// + /// Create a FlatBufferBuilder with a given initial size. + /// + /// + /// The initial size to use for the internal buffer. + /// + public FlatBufferBuilder(int initialSize) + { + if (initialSize <= 0) + throw new ArgumentOutOfRangeException("initialSize", + initialSize, "Must be greater than zero"); + _space = initialSize; + _bb = new ByteBuffer(initialSize); + } + + /// + /// Reset the FlatBufferBuilder by purging all data that it holds. + /// + public void Clear() + { + _space = _bb.Length; + _bb.Reset(); + _minAlign = 1; + while (_vtableSize > 0) _vtable[--_vtableSize] = 0; + _vtableSize = -1; + _objectStart = 0; + _numVtables = 0; + _vectorNumElems = 0; + } + + /// + /// Gets and sets a Boolean to disable the optimization when serializing + /// default values to a Table. + /// + /// In order to save space, fields that are set to their default value + /// don't get serialized into the buffer. + /// + public bool ForceDefaults { get; set; } + + /// @cond FLATBUFFERS_INTERNAL + + public int Offset { get { return _bb.Length - _space; } } + + public void Pad(int size) + { + _bb.PutByte(_space -= size, 0, size); + } + + // Doubles the size of the ByteBuffer, and copies the old data towards + // the end of the new buffer (since we build the buffer backwards). + void GrowBuffer() + { + _bb.GrowFront(_bb.Length << 1); + } + + // Prepare to write an element of `size` after `additional_bytes` + // have been written, e.g. if you write a string, you need to align + // such the int length field is aligned to SIZEOF_INT, and the string + // data follows it directly. + // If all you need to do is align, `additional_bytes` will be 0. + public void Prep(int size, int additionalBytes) + { + // Track the biggest thing we've ever aligned to. + if (size > _minAlign) + _minAlign = size; + // Find the amount of alignment needed such that `size` is properly + // aligned after `additional_bytes` + var alignSize = + ((~((int)_bb.Length - _space + additionalBytes)) + 1) & + (size - 1); + // Reallocate the buffer if needed. + while (_space < alignSize + size + additionalBytes) + { + var oldBufSize = (int)_bb.Length; + GrowBuffer(); + _space += (int)_bb.Length - oldBufSize; + + } + if (alignSize > 0) + Pad(alignSize); + } + + public void PutBool(bool x) + { + _bb.PutByte(_space -= sizeof(byte), (byte)(x ? 1 : 0)); + } + + public void PutSbyte(sbyte x) + { + _bb.PutSbyte(_space -= sizeof(sbyte), x); + } + + public void PutByte(byte x) + { + _bb.PutByte(_space -= sizeof(byte), x); + } + + public void PutShort(short x) + { + _bb.PutShort(_space -= sizeof(short), x); + } + + public void PutUshort(ushort x) + { + _bb.PutUshort(_space -= sizeof(ushort), x); + } + + public void PutInt(int x) + { + _bb.PutInt(_space -= sizeof(int), x); + } + + public void PutUint(uint x) + { + _bb.PutUint(_space -= sizeof(uint), x); + } + + public void PutLong(long x) + { + _bb.PutLong(_space -= sizeof(long), x); + } + + public void PutUlong(ulong x) + { + _bb.PutUlong(_space -= sizeof(ulong), x); + } + + public void PutFloat(float x) + { + _bb.PutFloat(_space -= sizeof(float), x); + } + + /// + /// Puts an array of type T into this builder at the + /// current offset + /// + /// The type of the input data + /// The array to copy data from + public void Put(T[] x) + where T : struct + { + _space = _bb.Put(_space, x); + } + + public void PutDouble(double x) + { + _bb.PutDouble(_space -= sizeof(double), x); + } + /// @endcond + + /// + /// Add a `bool` to the buffer (aligns the data and grows if necessary). + /// + /// The `bool` to add to the buffer. + public void AddBool(bool x) { Prep(sizeof(byte), 0); PutBool(x); } + + /// + /// Add a `sbyte` to the buffer (aligns the data and grows if necessary). + /// + /// The `sbyte` to add to the buffer. + public void AddSbyte(sbyte x) { Prep(sizeof(sbyte), 0); PutSbyte(x); } + + /// + /// Add a `byte` to the buffer (aligns the data and grows if necessary). + /// + /// The `byte` to add to the buffer. + public void AddByte(byte x) { Prep(sizeof(byte), 0); PutByte(x); } + + /// + /// Add a `short` to the buffer (aligns the data and grows if necessary). + /// + /// The `short` to add to the buffer. + public void AddShort(short x) { Prep(sizeof(short), 0); PutShort(x); } + + /// + /// Add an `ushort` to the buffer (aligns the data and grows if necessary). + /// + /// The `ushort` to add to the buffer. + public void AddUshort(ushort x) { Prep(sizeof(ushort), 0); PutUshort(x); } + + /// + /// Add an `int` to the buffer (aligns the data and grows if necessary). + /// + /// The `int` to add to the buffer. + public void AddInt(int x) { Prep(sizeof(int), 0); PutInt(x); } + + /// + /// Add an `uint` to the buffer (aligns the data and grows if necessary). + /// + /// The `uint` to add to the buffer. + public void AddUint(uint x) { Prep(sizeof(uint), 0); PutUint(x); } + + /// + /// Add a `long` to the buffer (aligns the data and grows if necessary). + /// + /// The `long` to add to the buffer. + public void AddLong(long x) { Prep(sizeof(long), 0); PutLong(x); } + + /// + /// Add an `ulong` to the buffer (aligns the data and grows if necessary). + /// + /// The `ulong` to add to the buffer. + public void AddUlong(ulong x) { Prep(sizeof(ulong), 0); PutUlong(x); } + + /// + /// Add a `float` to the buffer (aligns the data and grows if necessary). + /// + /// The `float` to add to the buffer. + public void AddFloat(float x) { Prep(sizeof(float), 0); PutFloat(x); } + + /// + /// Add an array of type T to the buffer (aligns the data and grows if necessary). + /// + /// The type of the input data + /// The array to copy data from + public void Add(T[] x) + where T : struct + { + if (x == null) + { + throw new ArgumentNullException("Cannot add a null array"); + } + + if( x.Length == 0) + { + // don't do anything if the array is empty + return; + } + + if(!ByteBuffer.IsSupportedType()) + { + throw new ArgumentException("Cannot add this Type array to the builder"); + } + + int size = ByteBuffer.SizeOf(); + // Need to prep on size (for data alignment) and then we pass the + // rest of the length (minus 1) as additional bytes + Prep(size, size * (x.Length - 1)); + Put(x); + } + + /// + /// Add a `double` to the buffer (aligns the data and grows if necessary). + /// + /// The `double` to add to the buffer. + public void AddDouble(double x) { Prep(sizeof(double), 0); + PutDouble(x); } + + /// + /// Adds an offset, relative to where it will be written. + /// + /// The offset to add to the buffer. + public void AddOffset(int off) + { + Prep(sizeof(int), 0); // Ensure alignment is already done. + if (off > Offset) + throw new ArgumentException(); + + off = Offset - off + sizeof(int); + PutInt(off); + } + + /// @cond FLATBUFFERS_INTERNAL + public void StartVector(int elemSize, int count, int alignment) + { + NotNested(); + _vectorNumElems = count; + Prep(sizeof(int), elemSize * count); + Prep(alignment, elemSize * count); // Just in case alignment > int. + } + /// @endcond + + /// + /// Writes data necessary to finish a vector construction. + /// + public VectorOffset EndVector() + { + PutInt(_vectorNumElems); + return new VectorOffset(Offset); + } + + /// + /// Creates a vector of tables. + /// + /// Offsets of the tables. + public VectorOffset CreateVectorOfTables(Offset[] offsets) where T : struct + { + NotNested(); + StartVector(sizeof(int), offsets.Length, sizeof(int)); + for (int i = offsets.Length - 1; i >= 0; i--) AddOffset(offsets[i].Value); + return EndVector(); + } + + /// @cond FLATBUFFERS_INTENRAL + public void Nested(int obj) + { + // Structs are always stored inline, so need to be created right + // where they are used. You'll get this assert if you created it + // elsewhere. + if (obj != Offset) + throw new Exception( + "FlatBuffers: struct must be serialized inline."); + } + + public void NotNested() + { + // You should not be creating any other objects or strings/vectors + // while an object is being constructed + if (_vtableSize >= 0) + throw new Exception( + "FlatBuffers: object serialization must not be nested."); + } + + public void StartObject(int numfields) + { + if (numfields < 0) + throw new ArgumentOutOfRangeException("Flatbuffers: invalid numfields"); + + NotNested(); + + if (_vtable.Length < numfields) + _vtable = new int[numfields]; + + _vtableSize = numfields; + _objectStart = Offset; + } + + + // Set the current vtable at `voffset` to the current location in the + // buffer. + public void Slot(int voffset) + { + if (voffset >= _vtableSize) + throw new IndexOutOfRangeException("Flatbuffers: invalid voffset"); + + _vtable[voffset] = Offset; + } + + /// + /// Adds a Boolean to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddBool(int o, bool x, bool d) { if (ForceDefaults || x != d) { AddBool(x); Slot(o); } } + + /// + /// Adds a SByte to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddSbyte(int o, sbyte x, sbyte d) { if (ForceDefaults || x != d) { AddSbyte(x); Slot(o); } } + + /// + /// Adds a Byte to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddByte(int o, byte x, byte d) { if (ForceDefaults || x != d) { AddByte(x); Slot(o); } } + + /// + /// Adds a Int16 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddShort(int o, short x, int d) { if (ForceDefaults || x != d) { AddShort(x); Slot(o); } } + + /// + /// Adds a UInt16 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddUshort(int o, ushort x, ushort d) { if (ForceDefaults || x != d) { AddUshort(x); Slot(o); } } + + /// + /// Adds an Int32 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddInt(int o, int x, int d) { if (ForceDefaults || x != d) { AddInt(x); Slot(o); } } + + /// + /// Adds a UInt32 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddUint(int o, uint x, uint d) { if (ForceDefaults || x != d) { AddUint(x); Slot(o); } } + + /// + /// Adds an Int64 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddLong(int o, long x, long d) { if (ForceDefaults || x != d) { AddLong(x); Slot(o); } } + + /// + /// Adds a UInt64 to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddUlong(int o, ulong x, ulong d) { if (ForceDefaults || x != d) { AddUlong(x); Slot(o); } } + + /// + /// Adds a Single to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddFloat(int o, float x, double d) { if (ForceDefaults || x != d) { AddFloat(x); Slot(o); } } + + /// + /// Adds a Double to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddDouble(int o, double x, double d) { if (ForceDefaults || x != d) { AddDouble(x); Slot(o); } } + + /// + /// Adds a buffer offset to the Table at index `o` in its vtable using the value `x` and default `d` + /// + /// The index into the vtable + /// The value to put into the buffer. If the value is equal to the default + /// and is false, the value will be skipped. + /// The default value to compare the value against + public void AddOffset(int o, int x, int d) { if (ForceDefaults || x != d) { AddOffset(x); Slot(o); } } + /// @endcond + + /// + /// Encode the string `s` in the buffer using UTF-8. + /// + /// The string to encode. + /// + /// The offset in the buffer where the encoded string starts. + /// + public StringOffset CreateString(string s) + { + NotNested(); + AddByte(0); + var utf8StringLen = Encoding.UTF8.GetByteCount(s); + StartVector(1, utf8StringLen, 1); + _bb.PutStringUTF8(_space -= utf8StringLen, s); + return new StringOffset(EndVector().Value); + } + + /// @cond FLATBUFFERS_INTERNAL + // Structs are stored inline, so nothing additional is being added. + // `d` is always 0. + public void AddStruct(int voffset, int x, int d) + { + if (x != d) + { + Nested(x); + Slot(voffset); + } + } + + public int EndObject() + { + if (_vtableSize < 0) + throw new InvalidOperationException( + "Flatbuffers: calling endObject without a startObject"); + + AddInt((int)0); + var vtableloc = Offset; + // Write out the current vtable. + int i = _vtableSize - 1; + // Trim trailing zeroes. + for (; i >= 0 && _vtable[i] == 0; i--) {} + int trimmedSize = i + 1; + for (; i >= 0 ; i--) { + // Offset relative to the start of the table. + short off = (short)(_vtable[i] != 0 + ? vtableloc - _vtable[i] + : 0); + AddShort(off); + + // clear out written entry + _vtable[i] = 0; + } + + const int standardFields = 2; // The fields below: + AddShort((short)(vtableloc - _objectStart)); + AddShort((short)((trimmedSize + standardFields) * + sizeof(short))); + + // Search for an existing vtable that matches the current one. + int existingVtable = 0; + for (i = 0; i < _numVtables; i++) { + int vt1 = _bb.Length - _vtables[i]; + int vt2 = _space; + short len = _bb.GetShort(vt1); + if (len == _bb.GetShort(vt2)) { + for (int j = sizeof(short); j < len; j += sizeof(short)) { + if (_bb.GetShort(vt1 + j) != _bb.GetShort(vt2 + j)) { + goto endLoop; + } + } + existingVtable = _vtables[i]; + break; + } + + endLoop: { } + } + + if (existingVtable != 0) { + // Found a match: + // Remove the current vtable. + _space = _bb.Length - vtableloc; + // Point table to existing vtable. + _bb.PutInt(_space, existingVtable - vtableloc); + } else { + // No match: + // Add the location of the current vtable to the list of + // vtables. + if (_numVtables == _vtables.Length) + { + // Arrays.CopyOf(vtables num_vtables * 2); + var newvtables = new int[ _numVtables * 2]; + Array.Copy(_vtables, newvtables, _vtables.Length); + + _vtables = newvtables; + }; + _vtables[_numVtables++] = Offset; + // Point table to current vtable. + _bb.PutInt(_bb.Length - vtableloc, Offset - vtableloc); + } + + _vtableSize = -1; + return vtableloc; + } + + // This checks a required field has been set in a given table that has + // just been constructed. + public void Required(int table, int field) + { + int table_start = _bb.Length - table; + int vtable_start = table_start - _bb.GetInt(table_start); + bool ok = _bb.GetShort(vtable_start + field) != 0; + // If this fails, the caller will show what field needs to be set. + if (!ok) + throw new InvalidOperationException("FlatBuffers: field " + field + + " must be set"); + } + /// @endcond + + /// + /// Finalize a buffer, pointing to the given `root_table`. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// Whether to prefix the size to the buffer. + /// + protected void Finish(int rootTable, bool sizePrefix) + { + Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0)); + AddOffset(rootTable); + if (sizePrefix) { + AddInt(_bb.Length - _space); + } + _bb.Position = _space; + } + + /// + /// Finalize a buffer, pointing to the given `root_table`. + /// + /// + /// An offset to be added to the buffer. + /// + public void Finish(int rootTable) + { + Finish(rootTable, false); + } + + /// + /// Finalize a buffer, pointing to the given `root_table`, with the size prefixed. + /// + /// + /// An offset to be added to the buffer. + /// + public void FinishSizePrefixed(int rootTable) + { + Finish(rootTable, true); + } + + /// + /// Get the ByteBuffer representing the FlatBuffer. + /// + /// + /// This is typically only called after you call `Finish()`. + /// The actual data starts at the ByteBuffer's current position, + /// not necessarily at `0`. + /// + /// + /// Returns the ByteBuffer for this FlatBuffer. + /// + public ByteBuffer DataBuffer { get { return _bb; } } + + /// + /// A utility function to copy and return the ByteBuffer data as a + /// `byte[]`. + /// + /// + /// A full copy of the FlatBuffer data. + /// + public byte[] SizedByteArray() + { + return _bb.ToSizedArray(); + } + + /// + /// Finalize a buffer, pointing to the given `rootTable`. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// + /// + /// Whether to prefix the size to the buffer. + /// + protected void Finish(int rootTable, string fileIdentifier, bool sizePrefix) + { + Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0) + + FlatBufferConstants.FileIdentifierLength); + if (fileIdentifier.Length != + FlatBufferConstants.FileIdentifierLength) + throw new ArgumentException( + "FlatBuffers: file identifier must be length " + + FlatBufferConstants.FileIdentifierLength, + "fileIdentifier"); + for (int i = FlatBufferConstants.FileIdentifierLength - 1; i >= 0; + i--) + { + AddByte((byte)fileIdentifier[i]); + } + Finish(rootTable, sizePrefix); + } + + /// + /// Finalize a buffer, pointing to the given `rootTable`. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// + public void Finish(int rootTable, string fileIdentifier) + { + Finish(rootTable, fileIdentifier, false); + } + + /// + /// Finalize a buffer, pointing to the given `rootTable`, with the size prefixed. + /// + /// + /// An offset to be added to the buffer. + /// + /// + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// + public void FinishSizePrefixed(int rootTable, string fileIdentifier) + { + Finish(rootTable, fileIdentifier, true); + } + } +} + +/// @} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs new file mode 100644 index 0000000000000..e30f3f3944f11 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs @@ -0,0 +1,29 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace FlatBuffers +{ + public static class FlatBufferConstants + { + public const int FileIdentifierLength = 4; + public const int SizePrefixLength = 4; + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs new file mode 100644 index 0000000000000..6a15aba6e5b6a --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs @@ -0,0 +1,28 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace FlatBuffers +{ + /// + /// This is the base for both structs and tables. + /// + public interface IFlatbufferObject + { + void __init(int _i, ByteBuffer _bb); + + ByteBuffer ByteBuffer { get; } + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs new file mode 100644 index 0000000000000..2b17cec85d4bd --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs @@ -0,0 +1,48 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace FlatBuffers +{ + /// + /// Offset class for typesafe assignments. + /// + public struct Offset where T : struct + { + public int Value; + public Offset(int value) + { + Value = value; + } + } + + public struct StringOffset + { + public int Value; + public StringOffset(int value) + { + Value = value; + } + } + + public struct VectorOffset + { + public int Value; + public VectorOffset(int value) + { + Value = value; + } + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs new file mode 100644 index 0000000000000..61da32f77bced --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs @@ -0,0 +1,27 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace FlatBuffers +{ + /// + /// All structs in the generated code derive from this class, and add their own accessors. + /// + public struct Struct + { + public int bb_pos; + public ByteBuffer bb; + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs new file mode 100644 index 0000000000000..07db5f423151d --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs @@ -0,0 +1,177 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Text; + +namespace FlatBuffers +{ + /// + /// All tables in the generated code derive from this struct, and add their own accessors. + /// + public struct Table + { + public int bb_pos; + public ByteBuffer bb; + + public ByteBuffer ByteBuffer { get { return bb; } } + + // Look up a field in the vtable, return an offset into the object, or 0 if the field is not + // present. + public int __offset(int vtableOffset) + { + int vtable = bb_pos - bb.GetInt(bb_pos); + return vtableOffset < bb.GetShort(vtable) ? (int)bb.GetShort(vtable + vtableOffset) : 0; + } + + public static int __offset(int vtableOffset, int offset, ByteBuffer bb) + { + int vtable = bb.Length - offset; + return (int)bb.GetShort(vtable + vtableOffset - bb.GetInt(vtable)) + vtable; + } + + // Retrieve the relative offset stored at "offset" + public int __indirect(int offset) + { + return offset + bb.GetInt(offset); + } + + public static int __indirect(int offset, ByteBuffer bb) + { + return offset + bb.GetInt(offset); + } + + // Create a .NET String from UTF-8 data stored inside the flatbuffer. + public string __string(int offset) + { + offset += bb.GetInt(offset); + var len = bb.GetInt(offset); + var startPos = offset + sizeof(int); + return bb.GetStringUTF8(startPos, len); + } + + // Get the length of a vector whose offset is stored at "offset" in this object. + public int __vector_len(int offset) + { + offset += bb_pos; + offset += bb.GetInt(offset); + return bb.GetInt(offset); + } + + // Get the start of data of a vector whose offset is stored at "offset" in this object. + public int __vector(int offset) + { + offset += bb_pos; + return offset + bb.GetInt(offset) + sizeof(int); // data starts after the length + } + + // Get the data of a vector whoses offset is stored at "offset" in this object as an + // ArraySegment<byte>. If the vector is not present in the ByteBuffer, + // then a null value will be returned. + public ArraySegment? __vector_as_arraysegment(int offset) + { + var o = this.__offset(offset); + if (0 == o) + { + return null; + } + + var pos = this.__vector(o); + var len = this.__vector_len(o); + return bb.ToArraySegment(pos, len); + } + + // Get the data of a vector whoses offset is stored at "offset" in this object as an + // T[]. If the vector is not present in the ByteBuffer, then a null value will be + // returned. + public T[] __vector_as_array(int offset) + where T : struct + { + if(!BitConverter.IsLittleEndian) + { + throw new NotSupportedException("Getting typed arrays on a Big Endian " + + "system is not support"); + } + + var o = this.__offset(offset); + if (0 == o) + { + return null; + } + + var pos = this.__vector(o); + var len = this.__vector_len(o); + return bb.ToArray(pos, len); + } + + // Initialize any Table-derived type to point to the union at the given offset. + public T __union(int offset) where T : struct, IFlatbufferObject + { + offset += bb_pos; + T t = new T(); + t.__init(offset + bb.GetInt(offset), bb); + return t; + } + + public static bool __has_identifier(ByteBuffer bb, string ident) + { + if (ident.Length != FlatBufferConstants.FileIdentifierLength) + throw new ArgumentException("FlatBuffers: file identifier must be length " + FlatBufferConstants.FileIdentifierLength, "ident"); + + for (var i = 0; i < FlatBufferConstants.FileIdentifierLength; i++) + { + if (ident[i] != (char)bb.Get(bb.Position + sizeof(int) + i)) return false; + } + + return true; + } + + // Compare strings in the ByteBuffer. + public static int CompareStrings(int offset_1, int offset_2, ByteBuffer bb) + { + offset_1 += bb.GetInt(offset_1); + offset_2 += bb.GetInt(offset_2); + var len_1 = bb.GetInt(offset_1); + var len_2 = bb.GetInt(offset_2); + var startPos_1 = offset_1 + sizeof(int); + var startPos_2 = offset_2 + sizeof(int); + var len = Math.Min(len_1, len_2); + for(int i = 0; i < len; i++) { + byte b1 = bb.Get(i + startPos_1); + byte b2 = bb.Get(i + startPos_2); + if (b1 != b2) + return b1 - b2; + } + return len_1 - len_2; + } + + // Compare string from the ByteBuffer with the string object + public static int CompareStrings(int offset_1, byte[] key, ByteBuffer bb) + { + offset_1 += bb.GetInt(offset_1); + var len_1 = bb.GetInt(offset_1); + var len_2 = key.Length; + var startPos_1 = offset_1 + sizeof(int); + var len = Math.Min(len_1, len_2); + for (int i = 0; i < len; i++) { + byte b = bb.Get(i + startPos_1); + if (b != key[i]) + return b - key[i]; + } + return len_1 - len_2; + } + } +} diff --git a/csharp/src/Apache.Arrow/Flatbuf/Footer.cs b/csharp/src/Apache.Arrow/Flatbuf/Footer.cs new file mode 100644 index 0000000000000..f45790a1ca5b7 --- /dev/null +++ b/csharp/src/Apache.Arrow/Flatbuf/Footer.cs @@ -0,0 +1,59 @@ +// +// automatically generated by the FlatBuffers compiler, do not modify +// + +namespace Apache.Arrow.Flatbuf +{ + +using global::System; +using global::FlatBuffers; + +/// ---------------------------------------------------------------------- +/// Arrow File metadata +/// +public struct Footer : IFlatbufferObject +{ + private Table __p; + public ByteBuffer ByteBuffer { get { return __p.bb; } } + public static Footer GetRootAsFooter(ByteBuffer _bb) { return GetRootAsFooter(_bb, new Footer()); } + public static Footer GetRootAsFooter(ByteBuffer _bb, Footer obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } + public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; } + public Footer __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } + + public MetadataVersion Version { get { int o = __p.__offset(4); return o != 0 ? (MetadataVersion)__p.bb.GetShort(o + __p.bb_pos) : MetadataVersion.V1; } } + public Schema? Schema { get { int o = __p.__offset(6); return o != 0 ? (Schema?)(new Schema()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } } + public Block? Dictionaries(int j) { int o = __p.__offset(8); return o != 0 ? (Block?)(new Block()).__assign(__p.__vector(o) + j * 24, __p.bb) : null; } + public int DictionariesLength { get { int o = __p.__offset(8); return o != 0 ? __p.__vector_len(o) : 0; } } + public Block? RecordBatches(int j) { int o = __p.__offset(10); return o != 0 ? (Block?)(new Block()).__assign(__p.__vector(o) + j * 24, __p.bb) : null; } + public int RecordBatchesLength { get { int o = __p.__offset(10); return o != 0 ? __p.__vector_len(o) : 0; } } + + public static Offset