From 7c72b933f4aa27400db3119169648ef1b2a3e72c Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Tue, 13 Jun 2023 15:04:34 -0600 Subject: [PATCH 1/7] feat: add helper functions and documentation --- .gitignore | 4 + README.md | 27 ++++-- generate_google_protos.sh | 2 +- lib/google_protos/any.pb.ex | 3 +- lib/google_protos/duration.ex | 68 +++++++++++++++ lib/google_protos/duration.pb.ex | 7 -- lib/google_protos/empty.pb.ex | 3 +- lib/google_protos/field_mask.pb.ex | 3 +- lib/google_protos/struct.ex | 121 ++++++++++++++++++++++++++ lib/google_protos/struct.pb.ex | 47 ---------- lib/google_protos/timestamp.ex | 77 ++++++++++++++++ lib/google_protos/timestamp.pb.ex | 7 -- lib/google_protos/wrappers.pb.ex | 27 ++++-- mix.exs | 15 +++- mix.lock | 14 +-- test/google_protos/duration_test.exs | 37 ++++++++ test/google_protos/struct_test.exs | 69 +++++++++++++++ test/google_protos/timestamp_test.exs | 19 ++++ 18 files changed, 459 insertions(+), 91 deletions(-) create mode 100644 lib/google_protos/duration.ex delete mode 100644 lib/google_protos/duration.pb.ex create mode 100644 lib/google_protos/struct.ex delete mode 100644 lib/google_protos/struct.pb.ex create mode 100644 lib/google_protos/timestamp.ex delete mode 100644 lib/google_protos/timestamp.pb.ex create mode 100644 test/google_protos/duration_test.exs create mode 100644 test/google_protos/struct_test.exs create mode 100644 test/google_protos/timestamp_test.exs diff --git a/.gitignore b/.gitignore index d40d812..1e7074d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ erl_crash.dump # Ignore package tarball (built via "mix hex.build"). google_protobuf-*.tar +# Ignore generated protobufs that have been enhanced and overwritten. +lib/google_protos/duration.pb.ex +lib/google_protos/struct.pb.ex +lib/google_protos/timestamp.pb.ex diff --git a/README.md b/README.md index 8b05aa7..ceb2b63 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,27 @@ # Google Protos -For Elixir files generated from [Google's protobuf files](https://github.com/google/protobuf/tree/master/src/google/protobuf) using [protobuf-elixir](https://github.com/tony612/protobuf-elixir). +For Elixir files generated from [Google's protobuf files](https://github.com/google/protobuf/tree/master/src/google/protobuf) using [protobuf-elixir](https://github.com/elixir-protobuf/protobuf). -## How-To +By including `google_protos` in your `mix.exs`, you'll have access to _some_ of the `google.protobuf.*` modules already compiled to Elixir. This can help simplify your protobuf compiling pipeline. For your benefit, we also include some helper functions for converting the protobuf structs to native Elixir modules. See the documentation for more details. -### Convert `Google.Protobuf.Timestamp` to `DateTime` +## Docuemntation -1. Convert timestamp seconds to nanoseconds. -2. Add the timestamp nanos. -3. Convert to `DateTime` using `DateTime.from_unix!/2`. +Docuemntation is available on [hex.pm](https://hexdocs.pm/google_protos/). + +## Usage + +Include the `:google_protos` package in your `mix.exs` dependency list: ```elixir -timestamp = Google.Protobuf.Timestamp.new(seconds: 5, nanos: 100) -DateTime.from_unix!(timestamp.seconds * 1_000_000_000 + timestamp.nanos, :nanosecond) +defp deps do + [ + {:google_protos, "~> 0.3"} + ] +end ``` + +Use the `Google.Protobuf` modules. See [google_protos](https://hexdocs.pm/google_protos/) and [Elixir protobuf](https://hexdocs.pm/protobuf) documentation for more details. + +## Limits + +The Google Protos repository will only compile official Google protobuf messages, and will only include helper functions for _native_ Elixir modules. We do not accept PRs adding support for third party libraries (ecto, decimal, money, etc.) If there is a Google Protobuf message that maps nicely to an Elixir module where we don't include helpers, please feel free to open a PR. diff --git a/generate_google_protos.sh b/generate_google_protos.sh index b537c26..2b60739 100755 --- a/generate_google_protos.sh +++ b/generate_google_protos.sh @@ -10,7 +10,7 @@ PROTOS=(" protobuf/src/google/protobuf/wrappers.proto ") -rm -rf ./lib/google_protos/* +rm -rf ./lib/google_protos/*.pb.ex for file in $PROTOS; do protoc -I ./protobuf/src/google/protobuf/ --elixir_out=plugins=grpc:./lib/google_protos $file diff --git a/lib/google_protos/any.pb.ex b/lib/google_protos/any.pb.ex index e7fc918..7472c29 100644 --- a/lib/google_protos/any.pb.ex +++ b/lib/google_protos/any.pb.ex @@ -1,6 +1,7 @@ defmodule Google.Protobuf.Any do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :type_url, 1, type: :string, json_name: "typeUrl" field :value, 2, type: :bytes diff --git a/lib/google_protos/duration.ex b/lib/google_protos/duration.ex new file mode 100644 index 0000000..e5f3a42 --- /dev/null +++ b/lib/google_protos/duration.ex @@ -0,0 +1,68 @@ +defmodule Google.Protobuf.Duration do + @moduledoc """ + A Duration represents a signed, fixed-length span of time represented + as a count of seconds and fractions of seconds at nanosecond + resolution. It is independent of any calendar and concepts like "day" + or "month". It is related to Timestamp in that the difference between + two Timestamp values is a Duration and it can be added or subtracted + from a Timestamp. Range is approximately +-10,000 years. + + ## JSON Mapping + + In JSON format, the Duration type is encoded as a string rather than an + object, where the string ends in the suffix "s" (indicating seconds) and + is preceded by the number of seconds, with nanoseconds expressed as + fractional seconds. For example, 3 seconds with 0 nanoseconds should be + encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should + be expressed in JSON format as "3.000000001s", and 3 seconds and 1 + microsecond should be expressed in JSON format as "3.000001s". + """ + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :seconds, 1, type: :int64 + field :nanos, 2, type: :int32 + + @doc """ + Converts a `Google.Protobuf.Duration` to a value and `System.time_unit()`. + """ + @spec to_time_unit(__MODULE__.t()) :: {integer(), System.time_unit()} + def to_time_unit(%{seconds: seconds, nanos: 0}) do + {seconds, :second} + end + + def to_time_unit(%{seconds: seconds, nanos: nanos}) do + {seconds * 1_000_000_000 + nanos, :nanosecond} + end + + @doc """ + Converts a value and `System.time_unit()` to a `Google.Protobuf.Duration`. + """ + @spec from_time_unit(integer(), System.time_unit()) :: __MODULE__.t() + def from_time_unit(seconds, :second) do + struct(__MODULE__, %{ + seconds: seconds + }) + end + + def from_time_unit(millisecond, :millisecond) do + struct(__MODULE__, %{ + seconds: div(millisecond, 1_000), + nanos: rem(millisecond, 1_000) * 1_000_000 + }) + end + + def from_time_unit(millisecond, :microsecond) do + struct(__MODULE__, %{ + seconds: div(millisecond, 1_000_000), + nanos: rem(millisecond, 1_000_000) * 1_000 + }) + end + + def from_time_unit(millisecond, :nanosecond) do + struct(__MODULE__, %{ + seconds: div(millisecond, 1_000_000_000), + nanos: rem(millisecond, 1_000_000_000) + }) + end +end diff --git a/lib/google_protos/duration.pb.ex b/lib/google_protos/duration.pb.ex deleted file mode 100644 index 0de562c..0000000 --- a/lib/google_protos/duration.pb.ex +++ /dev/null @@ -1,7 +0,0 @@ -defmodule Google.Protobuf.Duration do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - - field :seconds, 1, type: :int64 - field :nanos, 2, type: :int32 -end diff --git a/lib/google_protos/empty.pb.ex b/lib/google_protos/empty.pb.ex index 74f7005..334a639 100644 --- a/lib/google_protos/empty.pb.ex +++ b/lib/google_protos/empty.pb.ex @@ -1,4 +1,5 @@ defmodule Google.Protobuf.Empty do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" end diff --git a/lib/google_protos/field_mask.pb.ex b/lib/google_protos/field_mask.pb.ex index 045d7d1..6950164 100644 --- a/lib/google_protos/field_mask.pb.ex +++ b/lib/google_protos/field_mask.pb.ex @@ -1,6 +1,7 @@ defmodule Google.Protobuf.FieldMask do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :paths, 1, repeated: true, type: :string end diff --git a/lib/google_protos/struct.ex b/lib/google_protos/struct.ex new file mode 100644 index 0000000..0760b5a --- /dev/null +++ b/lib/google_protos/struct.ex @@ -0,0 +1,121 @@ +defmodule Google.Protobuf.NullValue do + @moduledoc false + + use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :NULL_VALUE, 0 +end + +defmodule Google.Protobuf.Struct.FieldsEntry do + @moduledoc false + + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :key, 1, type: :string + field :value, 2, type: Google.Protobuf.Value +end + +defmodule Google.Protobuf.Value do + @moduledoc false + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + oneof :kind, 0 + + field :null_value, 1, + type: Google.Protobuf.NullValue, + json_name: "nullValue", + enum: true, + oneof: 0 + + field :number_value, 2, type: :double, json_name: "numberValue", oneof: 0 + field :string_value, 3, type: :string, json_name: "stringValue", oneof: 0 + field :bool_value, 4, type: :bool, json_name: "boolValue", oneof: 0 + field :struct_value, 5, type: Google.Protobuf.Struct, json_name: "structValue", oneof: 0 + field :list_value, 6, type: Google.Protobuf.ListValue, json_name: "listValue", oneof: 0 +end + +defmodule Google.Protobuf.ListValue do + @moduledoc false + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :values, 1, repeated: true, type: Google.Protobuf.Value +end + +defmodule Google.Protobuf.Struct do + @moduledoc """ + Struct represents a structured data value, consisting of fields + which map to dynamically typed values. In some languages, Struct + might be supported by a native representation. For example, in + scripting languages like JS a struct is represented as an + object. The details of that representation are described together + with the proto support for the language. + + The JSON representation for Struct is JSON object. + """ + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :fields, 1, repeated: true, type: Google.Protobuf.Struct.FieldsEntry, map: true + + @doc """ + Converts a `Google.Protobuf.Struct` to a `map()`. + """ + @spec to_map(__MODULE__.t()) :: map() + def to_map(struct) do + Map.new(struct.fields, fn {k, v} -> + {k, to_map_value(v)} + end) + end + + defp to_map_value(%Google.Protobuf.Value{kind: {:null_value, :NULL_VALUE}}), do: nil + defp to_map_value(%Google.Protobuf.Value{kind: {:number_value, value}}), do: value + defp to_map_value(%Google.Protobuf.Value{kind: {:string_value, value}}), do: value + defp to_map_value(%Google.Protobuf.Value{kind: {:bool_value, value}}), do: value + + defp to_map_value(%Google.Protobuf.Value{kind: {:struct_value, struct}}), + do: to_map(struct) + + defp to_map_value(%Google.Protobuf.Value{kind: {:list_value, %{values: values}}}), + do: Enum.map(values, &to_map_value/1) + + @doc """ + Converts a `map()` to a `Google.Protobuf.Struct`. + """ + @spec from_map(map()) :: __MODULE__.t() + def from_map(map) do + struct(__MODULE__, %{ + fields: + Map.new(map, fn {k, v} -> + {to_string(k), from_map_value(v)} + end) + }) + end + + defp from_map_value(nil) do + %Google.Protobuf.Value{kind: {:null_value, :NULL_VALUE}} + end + + defp from_map_value(value) when is_number(value) do + %Google.Protobuf.Value{kind: {:number_value, value}} + end + + defp from_map_value(value) when is_binary(value) do + %Google.Protobuf.Value{kind: {:string_value, value}} + end + + defp from_map_value(value) when is_boolean(value) do + %Google.Protobuf.Value{kind: {:bool_value, value}} + end + + defp from_map_value(value) when is_map(value) do + %Google.Protobuf.Value{kind: {:struct_value, from_map(value)}} + end + + defp from_map_value(value) when is_list(value) do + %Google.Protobuf.Value{ + kind: {:list_value, %Google.Protobuf.ListValue{values: Enum.map(value, &from_map_value/1)}} + } + end +end diff --git a/lib/google_protos/struct.pb.ex b/lib/google_protos/struct.pb.ex deleted file mode 100644 index 6053127..0000000 --- a/lib/google_protos/struct.pb.ex +++ /dev/null @@ -1,47 +0,0 @@ -defmodule Google.Protobuf.NullValue do - @moduledoc false - use Protobuf, enum: true, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - - field :NULL_VALUE, 0 -end - -defmodule Google.Protobuf.Struct.FieldsEntry do - @moduledoc false - use Protobuf, map: true, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - - field :key, 1, type: :string - field :value, 2, type: Google.Protobuf.Value -end - -defmodule Google.Protobuf.Struct do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - - field :fields, 1, repeated: true, type: Google.Protobuf.Struct.FieldsEntry, map: true -end - -defmodule Google.Protobuf.Value do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - - oneof :kind, 0 - - field :null_value, 1, - type: Google.Protobuf.NullValue, - json_name: "nullValue", - enum: true, - oneof: 0 - - field :number_value, 2, type: :double, json_name: "numberValue", oneof: 0 - field :string_value, 3, type: :string, json_name: "stringValue", oneof: 0 - field :bool_value, 4, type: :bool, json_name: "boolValue", oneof: 0 - field :struct_value, 5, type: Google.Protobuf.Struct, json_name: "structValue", oneof: 0 - field :list_value, 6, type: Google.Protobuf.ListValue, json_name: "listValue", oneof: 0 -end - -defmodule Google.Protobuf.ListValue do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - - field :values, 1, repeated: true, type: Google.Protobuf.Value -end diff --git a/lib/google_protos/timestamp.ex b/lib/google_protos/timestamp.ex new file mode 100644 index 0000000..2e8f4d7 --- /dev/null +++ b/lib/google_protos/timestamp.ex @@ -0,0 +1,77 @@ +defmodule Google.Protobuf.Timestamp do + @moduledoc """ + A Timestamp represents a point in time independent of any time zone or local + calendar, encoded as a count of seconds and fractions of seconds at + nanosecond resolution. The count is relative to an epoch at UTC midnight on + January 1, 1970, in the proleptic Gregorian calendar which extends the + Gregorian calendar backwards to year one. + + All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap + second table is needed for interpretation, using a [24-hour linear + smear](https://developers.google.com/time/smear). + + The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By + restricting to that range, we ensure that we can convert to and from [RFC + 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. + + ## JSON Mapping + + In JSON format, the Timestamp type is encoded as a string in the + [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the + format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" + where {year} is always expressed using four digits while {month}, {day}, + {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional + seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), + are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone + is required. A proto3 JSON serializer should always use UTC (as indicated by + "Z") when printing the Timestamp type and a proto3 JSON parser should be + able to accept both UTC and other timezones (as indicated by an offset). + + For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past + 01:30 UTC on January 15, 2017. + """ + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :seconds, 1, type: :int64 + field :nanos, 2, type: :int32 + + @doc """ + Converts a `DateTime` struct to a `#{__MODULE__}` struct. + + Note: Elixir `DateTime.from_unix!/2` will convert units to + microseconds internally. To keep the nanosecond precision, + we manually change the `DateTime` `:microsecond` precision. + + ## Examples + + iex> %Google.Protobuf.Timestamp{seconds: 5, nanos: 0} + ...> |> Google.Protobuf.Timestamp.to_datetime() + ~U[1970-01-01 00:00:05.000000Z] + + """ + @spec to_datetime(__MODULE__.t()) :: DateTime.t() + def to_datetime(%{seconds: seconds, nanos: nanos}) do + DateTime.from_unix!(seconds * 1_000_000_000 + nanos, :nanosecond) + end + + @doc """ + Converts a `#{__MODULE__}` struct to a `DateTime` struct. + + ## Examples + + iex> ~U[1970-01-01 00:00:05.000000Z] + ...> |> Google.Protobuf.Timestamp.from_datetime() + %Google.Protobuf.Timestamp{seconds: 5, nanos: 0} + + """ + @spec from_datetime(DateTime.t()) :: __MODULE__.t() + def from_datetime(%DateTime{} = datetime) do + nanoseconds = DateTime.to_unix(datetime, :nanosecond) + + struct(__MODULE__, %{ + seconds: div(nanoseconds, 1_000_000_000), + nanos: rem(nanoseconds, 1_000_000_000) + }) + end +end diff --git a/lib/google_protos/timestamp.pb.ex b/lib/google_protos/timestamp.pb.ex deleted file mode 100644 index 96945b3..0000000 --- a/lib/google_protos/timestamp.pb.ex +++ /dev/null @@ -1,7 +0,0 @@ -defmodule Google.Protobuf.Timestamp do - @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 - - field :seconds, 1, type: :int64 - field :nanos, 2, type: :int32 -end diff --git a/lib/google_protos/wrappers.pb.ex b/lib/google_protos/wrappers.pb.ex index 8bdd62a..bd6eef2 100644 --- a/lib/google_protos/wrappers.pb.ex +++ b/lib/google_protos/wrappers.pb.ex @@ -1,62 +1,71 @@ defmodule Google.Protobuf.DoubleValue do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :double end defmodule Google.Protobuf.FloatValue do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :float end defmodule Google.Protobuf.Int64Value do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :int64 end defmodule Google.Protobuf.UInt64Value do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :uint64 end defmodule Google.Protobuf.Int32Value do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :int32 end defmodule Google.Protobuf.UInt32Value do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :uint32 end defmodule Google.Protobuf.BoolValue do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :bool end defmodule Google.Protobuf.StringValue do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :string end defmodule Google.Protobuf.BytesValue do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.10.0", syntax: :proto3 + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :bytes end diff --git a/mix.exs b/mix.exs index 037d830..cf18b08 100644 --- a/mix.exs +++ b/mix.exs @@ -9,6 +9,7 @@ defmodule GoogleProtos.MixProject do elixir: "~> 1.6", start_permanent: Mix.env() == :prod, deps: deps(), + docs: docs(), description: "Protos by Google", package: package() ] @@ -20,8 +21,18 @@ defmodule GoogleProtos.MixProject do defp deps do [ - {:protobuf, "~> 0.10"}, - {:ex_doc, ">= 0.0.0", only: :dev} + {:protobuf, "~> 0.12"}, + {:ex_doc, "~> 0.29", only: :dev}, + {:jason, "~> 1.4", only: :test} + ] + end + + defp docs do + [ + extras: [ + "README.md": [filename: "overview", title: "Overview"] + ], + main: "overview" ] end diff --git a/mix.lock b/mix.lock index 13cf6ea..7c64595 100644 --- a/mix.lock +++ b/mix.lock @@ -1,10 +1,10 @@ %{ - "earmark": {:hex, :earmark, "1.2.4", "99b637c62a4d65a20a9fb674b8cffb8baa771c04605a80c911c4418c69b75439", [:mix], [], "hexpm", "1b34655872366414f69dd987cb121c049f76984b6ac69f52fff6d8fd64d29cfd"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.17", "6f3c7e94170377ba45241d394389e800fb15adc5de51d0a3cd52ae766aafd63f", [:mix], [], "hexpm", "f93ac89c9feca61c165b264b5837bf82344d13bebc634cd575cb711e2e342023"}, - "ex_doc": {:hex, :ex_doc, "0.25.5", "ac3c5425a80b4b7c4dfecdf51fa9c23a44877124dd8ca34ee45ff608b1c6deb9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "688cfa538cdc146bc4291607764a7f1fcfa4cce8009ecd62de03b27197528350"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"}, + "ex_doc": {:hex, :ex_doc, "0.29.4", "6257ecbb20c7396b1fe5accd55b7b0d23f44b6aa18017b415cb4c2b91d997729", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2c6699a737ae46cb61e4ed012af931b57b699643b24dabe2400a8168414bc4f5"}, + "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, + "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"}, - "protobuf": {:hex, :protobuf, "0.10.0", "4e8e3cf64c5be203b329f88bb8b916cb8d00fb3a12b2ac1f545463ae963c869f", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4ae21a386142357aa3d31ccf5f7d290f03f3fa6f209755f6e87fc2c58c147893"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, + "protobuf": {:hex, :protobuf, "0.12.0", "58c0dfea5f929b96b5aa54ec02b7130688f09d2de5ddc521d696eec2a015b223", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "75fa6cbf262062073dd51be44dd0ab940500e18386a6c4e87d5819a58964dc45"}, } diff --git a/test/google_protos/duration_test.exs b/test/google_protos/duration_test.exs new file mode 100644 index 0000000..906983e --- /dev/null +++ b/test/google_protos/duration_test.exs @@ -0,0 +1,37 @@ +defmodule Google.Protobuf.DurationTest do + use ExUnit.Case + + alias Google.Protobuf.Duration + + describe "to_time_unit/1" do + test "converts to total seconds if no nanoseconds specified" do + assert {4200, :second} == Duration.to_time_unit(%Duration{seconds: 4200}) + end + + test "converts to total nanoseconds if specified" do + assert {20_000_000_100, :nanosecond} == + Duration.to_time_unit(%Duration{seconds: 20, nanos: 100}) + end + end + + describe "from_time_unit/2" do + test "converts :second to duration" do + assert %Duration{seconds: 11} == Duration.from_time_unit(11, :second) + end + + test "converts :millisecond to duration" do + assert %Duration{seconds: 11, nanos: 111_000_000} == + Duration.from_time_unit(11111, :millisecond) + end + + test "converts :microsecond to duration" do + assert %Duration{seconds: 11, nanos: 111_111_000} == + Duration.from_time_unit(11_111_111, :microsecond) + end + + test "converts :nanosecond to duration" do + assert %Duration{seconds: 11, nanos: 111_111_111} == + Duration.from_time_unit(11_111_111_111, :nanosecond) + end + end +end diff --git a/test/google_protos/struct_test.exs b/test/google_protos/struct_test.exs new file mode 100644 index 0000000..c688f86 --- /dev/null +++ b/test/google_protos/struct_test.exs @@ -0,0 +1,69 @@ +defmodule Google.Protobuf.StructTest do + use ExUnit.Case + + alias Google.Protobuf.Struct + + @basic_json """ + { + "key_one": "value_one", + "key_two": 1234, + "key_three": null, + "key_four": true + } + """ + + @basic_elixir %{ + "key_one" => "value_one", + "key_two" => 1234, + "key_three" => nil, + "key_four" => true + } + + @advanced_json """ + { + "key_two": [1, 2, 3, null, true, "value"], + "key_three": { + "key_four": "value_four", + "key_five": { + "key_six": 99, + "key_seven": { + "key_eight": "value_eight" + } + } + } + } + """ + + @advanced_elixir %{ + "key_two" => [1, 2, 3, nil, true, "value"], + "key_three" => %{ + "key_four" => "value_four", + "key_five" => %{ + "key_six" => 99, + "key_seven" => %{ + "key_eight" => "value_eight" + } + } + } + } + + describe "to_map/1" do + test "converts basic json to map" do + assert @basic_elixir == Struct.to_map(Protobuf.JSON.decode!(@basic_json, Struct)) + end + + test "converts advanced json to map" do + assert @advanced_elixir == Struct.to_map(Protobuf.JSON.decode!(@advanced_json, Struct)) + end + end + + describe "from_map/1" do + test "converts basic elixir to struct" do + assert Protobuf.JSON.decode!(@basic_json, Struct) == Struct.from_map(@basic_elixir) + end + + test "converts advanced elixir to struct" do + assert Protobuf.JSON.decode!(@advanced_json, Struct) == Struct.from_map(@advanced_elixir) + end + end +end diff --git a/test/google_protos/timestamp_test.exs b/test/google_protos/timestamp_test.exs new file mode 100644 index 0000000..5968449 --- /dev/null +++ b/test/google_protos/timestamp_test.exs @@ -0,0 +1,19 @@ +defmodule Google.Protobuf.TimestampTest do + use ExUnit.Case + + alias Google.Protobuf.Timestamp + + describe "to_datetime/1" do + test "converts to DateTime" do + assert ~U[1970-01-01 00:00:05.000000Z] == + Timestamp.to_datetime(%Timestamp{seconds: 5, nanos: 0}) + end + end + + describe "from_datetime/1" do + test "converts from DateTime" do + assert %Timestamp{seconds: 5, nanos: 0} == + Timestamp.from_datetime(~U[1970-01-01 00:00:05.000000Z]) + end + end +end From 3e37228836311efd23b3abc80b49f06dfed9fa32 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Tue, 13 Jun 2023 19:04:10 -0600 Subject: [PATCH 2/7] update docs --- lib/google_protos/duration.ex | 18 ++++++++++++++++++ lib/google_protos/struct.ex | 12 ++++++++++++ lib/google_protos/timestamp.ex | 17 ++++++++++------- test/google_protos/timestamp_test.exs | 6 ++++++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/lib/google_protos/duration.ex b/lib/google_protos/duration.ex index e5f3a42..48fb4e9 100644 --- a/lib/google_protos/duration.ex +++ b/lib/google_protos/duration.ex @@ -25,6 +25,15 @@ defmodule Google.Protobuf.Duration do @doc """ Converts a `Google.Protobuf.Duration` to a value and `System.time_unit()`. + + ## Examples + + iex> Duration.to_time_unit(%Duration{seconds: 10}) + {10, :second} + + iex> Duration.to_time_unit(%Duration{seconds: 20, nanos: 100}) + {20_000_000_100, :nanosecond} + """ @spec to_time_unit(__MODULE__.t()) :: {integer(), System.time_unit()} def to_time_unit(%{seconds: seconds, nanos: 0}) do @@ -37,6 +46,15 @@ defmodule Google.Protobuf.Duration do @doc """ Converts a value and `System.time_unit()` to a `Google.Protobuf.Duration`. + + ## Examples + + iex> Duration.from_time_unit(11, :second) + %Duration{seconds: 11} + + iex> Duration.from_time_unit(11_111_111, :microsecond) + %Duration{seconds: 11, nanos: 111_111_000} + """ @spec from_time_unit(integer(), System.time_unit()) :: __MODULE__.t() def from_time_unit(seconds, :second) do diff --git a/lib/google_protos/struct.ex b/lib/google_protos/struct.ex index 0760b5a..3240c26 100644 --- a/lib/google_protos/struct.ex +++ b/lib/google_protos/struct.ex @@ -61,6 +61,12 @@ defmodule Google.Protobuf.Struct do @doc """ Converts a `Google.Protobuf.Struct` to a `map()`. + + ## Examples + + iex> Struct.to_map(%Struct{}) + %{} + """ @spec to_map(__MODULE__.t()) :: map() def to_map(struct) do @@ -82,6 +88,12 @@ defmodule Google.Protobuf.Struct do @doc """ Converts a `map()` to a `Google.Protobuf.Struct`. + + ## Examples + + iex> Struct.from_map(%{key: "value"}) + %Struct{} + """ @spec from_map(map()) :: __MODULE__.t() def from_map(map) do diff --git a/lib/google_protos/timestamp.ex b/lib/google_protos/timestamp.ex index 2e8f4d7..9dd0b52 100644 --- a/lib/google_protos/timestamp.ex +++ b/lib/google_protos/timestamp.ex @@ -40,15 +40,19 @@ defmodule Google.Protobuf.Timestamp do Converts a `DateTime` struct to a `#{__MODULE__}` struct. Note: Elixir `DateTime.from_unix!/2` will convert units to - microseconds internally. To keep the nanosecond precision, - we manually change the `DateTime` `:microsecond` precision. + microseconds internally. Nanosecond precision is not guaranteed. + See examples for details. ## Examples - iex> %Google.Protobuf.Timestamp{seconds: 5, nanos: 0} - ...> |> Google.Protobuf.Timestamp.to_datetime() + iex> Timestamp.to_datetime(%Timestamp{seconds: 5, nanos: 0}) ~U[1970-01-01 00:00:05.000000Z] + iex> one = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 100}) + ...> two = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 105}) + ...> DateTime.diff(one, two, :nanosecond) + 0 + """ @spec to_datetime(__MODULE__.t()) :: DateTime.t() def to_datetime(%{seconds: seconds, nanos: nanos}) do @@ -60,9 +64,8 @@ defmodule Google.Protobuf.Timestamp do ## Examples - iex> ~U[1970-01-01 00:00:05.000000Z] - ...> |> Google.Protobuf.Timestamp.from_datetime() - %Google.Protobuf.Timestamp{seconds: 5, nanos: 0} + iex> Timestamp.from_datetime(~U[1970-01-01 00:00:05.000000Z]) + %Timestamp{seconds: 5, nanos: 0} """ @spec from_datetime(DateTime.t()) :: __MODULE__.t() diff --git a/test/google_protos/timestamp_test.exs b/test/google_protos/timestamp_test.exs index 5968449..6e3ae84 100644 --- a/test/google_protos/timestamp_test.exs +++ b/test/google_protos/timestamp_test.exs @@ -8,6 +8,12 @@ defmodule Google.Protobuf.TimestampTest do assert ~U[1970-01-01 00:00:05.000000Z] == Timestamp.to_datetime(%Timestamp{seconds: 5, nanos: 0}) end + + test "nanosecond precision" do + one = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 100}) + two = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 105}) + assert 0 == DateTime.diff(one, two, :nanosecond) + end end describe "from_datetime/1" do From 94e1e61eb149d29483864839e725ebee4be533b3 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Tue, 13 Jun 2023 22:42:55 -0600 Subject: [PATCH 3/7] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ceb2b63..ece4691 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ For Elixir files generated from [Google's protobuf files](https://github.com/goo By including `google_protos` in your `mix.exs`, you'll have access to _some_ of the `google.protobuf.*` modules already compiled to Elixir. This can help simplify your protobuf compiling pipeline. For your benefit, we also include some helper functions for converting the protobuf structs to native Elixir modules. See the documentation for more details. -## Docuemntation +## Documentation -Docuemntation is available on [hex.pm](https://hexdocs.pm/google_protos/). +Documentation is available on [hex.pm](https://hexdocs.pm/google_protos/). ## Usage From 4dac319ddb01f51e173cb79c8577f6482ad77eda Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Fri, 16 Jun 2023 10:52:08 -0600 Subject: [PATCH 4/7] template generated files with functions --- .formatter.exs | 2 +- .gitignore | 5 - generate_google_protos.sh | 16 +++ lib/google_protos/any.pb.ex | 2 +- .../{duration.ex => duration.pb.ex} | 21 +--- lib/google_protos/empty.pb.ex | 2 +- lib/google_protos/field_mask.pb.ex | 2 +- lib/google_protos/{struct.ex => struct.pb.ex} | 99 +++++++++---------- lib/google_protos/timestamp.ex | 80 --------------- lib/google_protos/timestamp.pb.ex | 50 ++++++++++ lib/google_protos/wrappers.pb.ex | 2 +- priv/templates/README.md | 9 ++ priv/templates/duration.pb.ex | 64 ++++++++++++ priv/templates/struct.pb.ex | 79 +++++++++++++++ priv/templates/timestamp.pb.ex | 45 +++++++++ 15 files changed, 317 insertions(+), 161 deletions(-) rename lib/google_protos/{duration.ex => duration.pb.ex} (64%) rename lib/google_protos/{struct.ex => struct.pb.ex} (64%) delete mode 100644 lib/google_protos/timestamp.ex create mode 100644 lib/google_protos/timestamp.pb.ex create mode 100644 priv/templates/README.md create mode 100644 priv/templates/duration.pb.ex create mode 100644 priv/templates/struct.pb.ex create mode 100644 priv/templates/timestamp.pb.ex diff --git a/.formatter.exs b/.formatter.exs index 8351d69..f22de05 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,4 @@ [ - inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"], + inputs: ["mix.exs", "{config,lib,priv,test}/**/*.{ex,exs}"], import_deps: [:protobuf] ] diff --git a/.gitignore b/.gitignore index 1e7074d..0798449 100644 --- a/.gitignore +++ b/.gitignore @@ -21,8 +21,3 @@ erl_crash.dump # Ignore package tarball (built via "mix hex.build"). google_protobuf-*.tar - -# Ignore generated protobufs that have been enhanced and overwritten. -lib/google_protos/duration.pb.ex -lib/google_protos/struct.pb.ex -lib/google_protos/timestamp.pb.ex diff --git a/generate_google_protos.sh b/generate_google_protos.sh index 2b60739..46b6c7d 100755 --- a/generate_google_protos.sh +++ b/generate_google_protos.sh @@ -15,3 +15,19 @@ rm -rf ./lib/google_protos/*.pb.ex for file in $PROTOS; do protoc -I ./protobuf/src/google/protobuf/ --elixir_out=plugins=grpc:./lib/google_protos $file done + +for template_file in priv/templates/*.pb.ex; do + file_name=$(basename "$template_file") + replacement_file="lib/google_protos/$file_name" + + if [ -f "$replacement_file" ]; then + for module in $(sed -n -E "s/defmodule\s(.+)\sdo/\1/p" "$template_file"); do + replacement_text=$(cat "$template_file" | sed -n "/^defmodule\s$module\sdo$/,/^end$/p" | sed 1,2d) + line_to_replace=$(cat -n "$replacement_file" | sed -n "/defmodule\s$module\sdo$/,/end$/p" | grep "end" | awk '{print $1}') + + export REPLACEMENT_TEXT=$replacement_text + perl -i -pe "s/end/\$ENV{\"REPLACEMENT_TEXT\"}/g if $. == $line_to_replace" "$replacement_file" + unset $REPLACEMENT_TEXT + done + fi +done diff --git a/lib/google_protos/any.pb.ex b/lib/google_protos/any.pb.ex index 7472c29..3d6c939 100644 --- a/lib/google_protos/any.pb.ex +++ b/lib/google_protos/any.pb.ex @@ -5,4 +5,4 @@ defmodule Google.Protobuf.Any do field :type_url, 1, type: :string, json_name: "typeUrl" field :value, 2, type: :bytes -end +end \ No newline at end of file diff --git a/lib/google_protos/duration.ex b/lib/google_protos/duration.pb.ex similarity index 64% rename from lib/google_protos/duration.ex rename to lib/google_protos/duration.pb.ex index 48fb4e9..96d1cc2 100644 --- a/lib/google_protos/duration.ex +++ b/lib/google_protos/duration.pb.ex @@ -1,22 +1,5 @@ defmodule Google.Protobuf.Duration do - @moduledoc """ - A Duration represents a signed, fixed-length span of time represented - as a count of seconds and fractions of seconds at nanosecond - resolution. It is independent of any calendar and concepts like "day" - or "month". It is related to Timestamp in that the difference between - two Timestamp values is a Duration and it can be added or subtracted - from a Timestamp. Range is approximately +-10,000 years. - - ## JSON Mapping - - In JSON format, the Duration type is encoded as a string rather than an - object, where the string ends in the suffix "s" (indicating seconds) and - is preceded by the number of seconds, with nanoseconds expressed as - fractional seconds. For example, 3 seconds with 0 nanoseconds should be - encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should - be expressed in JSON format as "3.000000001s", and 3 seconds and 1 - microsecond should be expressed in JSON format as "3.000001s". - """ + @moduledoc false use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" @@ -83,4 +66,4 @@ defmodule Google.Protobuf.Duration do nanos: rem(millisecond, 1_000_000_000) }) end -end +end \ No newline at end of file diff --git a/lib/google_protos/empty.pb.ex b/lib/google_protos/empty.pb.ex index 334a639..ded0331 100644 --- a/lib/google_protos/empty.pb.ex +++ b/lib/google_protos/empty.pb.ex @@ -2,4 +2,4 @@ defmodule Google.Protobuf.Empty do @moduledoc false use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" -end +end \ No newline at end of file diff --git a/lib/google_protos/field_mask.pb.ex b/lib/google_protos/field_mask.pb.ex index 6950164..d64b861 100644 --- a/lib/google_protos/field_mask.pb.ex +++ b/lib/google_protos/field_mask.pb.ex @@ -4,4 +4,4 @@ defmodule Google.Protobuf.FieldMask do use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :paths, 1, repeated: true, type: :string -end +end \ No newline at end of file diff --git a/lib/google_protos/struct.ex b/lib/google_protos/struct.pb.ex similarity index 64% rename from lib/google_protos/struct.ex rename to lib/google_protos/struct.pb.ex index 3240c26..d96ae9b 100644 --- a/lib/google_protos/struct.ex +++ b/lib/google_protos/struct.pb.ex @@ -15,45 +15,8 @@ defmodule Google.Protobuf.Struct.FieldsEntry do field :value, 2, type: Google.Protobuf.Value end -defmodule Google.Protobuf.Value do - @moduledoc false - - use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" - - oneof :kind, 0 - - field :null_value, 1, - type: Google.Protobuf.NullValue, - json_name: "nullValue", - enum: true, - oneof: 0 - - field :number_value, 2, type: :double, json_name: "numberValue", oneof: 0 - field :string_value, 3, type: :string, json_name: "stringValue", oneof: 0 - field :bool_value, 4, type: :bool, json_name: "boolValue", oneof: 0 - field :struct_value, 5, type: Google.Protobuf.Struct, json_name: "structValue", oneof: 0 - field :list_value, 6, type: Google.Protobuf.ListValue, json_name: "listValue", oneof: 0 -end - -defmodule Google.Protobuf.ListValue do - @moduledoc false - - use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" - - field :values, 1, repeated: true, type: Google.Protobuf.Value -end - defmodule Google.Protobuf.Struct do - @moduledoc """ - Struct represents a structured data value, consisting of fields - which map to dynamically typed values. In some languages, Struct - might be supported by a native representation. For example, in - scripting languages like JS a struct is represented as an - object. The details of that representation are described together - with the proto support for the language. - - The JSON representation for Struct is JSON object. - """ + @moduledoc false use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" @@ -75,15 +38,15 @@ defmodule Google.Protobuf.Struct do end) end - defp to_map_value(%Google.Protobuf.Value{kind: {:null_value, :NULL_VALUE}}), do: nil - defp to_map_value(%Google.Protobuf.Value{kind: {:number_value, value}}), do: value - defp to_map_value(%Google.Protobuf.Value{kind: {:string_value, value}}), do: value - defp to_map_value(%Google.Protobuf.Value{kind: {:bool_value, value}}), do: value + defp to_map_value(%{kind: {:null_value, :NULL_VALUE}}), do: nil + defp to_map_value(%{kind: {:number_value, value}}), do: value + defp to_map_value(%{kind: {:string_value, value}}), do: value + defp to_map_value(%{kind: {:bool_value, value}}), do: value - defp to_map_value(%Google.Protobuf.Value{kind: {:struct_value, struct}}), + defp to_map_value(%{kind: {:struct_value, struct}}), do: to_map(struct) - defp to_map_value(%Google.Protobuf.Value{kind: {:list_value, %{values: values}}}), + defp to_map_value(%{kind: {:list_value, %{values: values}}}), do: Enum.map(values, &to_map_value/1) @doc """ @@ -106,28 +69,60 @@ defmodule Google.Protobuf.Struct do end defp from_map_value(nil) do - %Google.Protobuf.Value{kind: {:null_value, :NULL_VALUE}} + struct(Google.Protobuf.Value, %{kind: {:null_value, :NULL_VALUE}}) end defp from_map_value(value) when is_number(value) do - %Google.Protobuf.Value{kind: {:number_value, value}} + struct(Google.Protobuf.Value, %{kind: {:number_value, value}}) end defp from_map_value(value) when is_binary(value) do - %Google.Protobuf.Value{kind: {:string_value, value}} + struct(Google.Protobuf.Value, %{kind: {:string_value, value}}) end defp from_map_value(value) when is_boolean(value) do - %Google.Protobuf.Value{kind: {:bool_value, value}} + struct(Google.Protobuf.Value, %{kind: {:bool_value, value}}) end defp from_map_value(value) when is_map(value) do - %Google.Protobuf.Value{kind: {:struct_value, from_map(value)}} + struct(Google.Protobuf.Value, %{kind: {:struct_value, from_map(value)}}) end defp from_map_value(value) when is_list(value) do - %Google.Protobuf.Value{ - kind: {:list_value, %Google.Protobuf.ListValue{values: Enum.map(value, &from_map_value/1)}} - } + struct(Google.Protobuf.Value, %{ + kind: + {:list_value, + struct(Google.Protobuf.ListValue, %{ + values: Enum.map(value, &from_map_value/1) + })} + }) end end + +defmodule Google.Protobuf.Value do + @moduledoc false + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + oneof :kind, 0 + + field :null_value, 1, + type: Google.Protobuf.NullValue, + json_name: "nullValue", + enum: true, + oneof: 0 + + field :number_value, 2, type: :double, json_name: "numberValue", oneof: 0 + field :string_value, 3, type: :string, json_name: "stringValue", oneof: 0 + field :bool_value, 4, type: :bool, json_name: "boolValue", oneof: 0 + field :struct_value, 5, type: Google.Protobuf.Struct, json_name: "structValue", oneof: 0 + field :list_value, 6, type: Google.Protobuf.ListValue, json_name: "listValue", oneof: 0 +end + +defmodule Google.Protobuf.ListValue do + @moduledoc false + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :values, 1, repeated: true, type: Google.Protobuf.Value +end \ No newline at end of file diff --git a/lib/google_protos/timestamp.ex b/lib/google_protos/timestamp.ex deleted file mode 100644 index 9dd0b52..0000000 --- a/lib/google_protos/timestamp.ex +++ /dev/null @@ -1,80 +0,0 @@ -defmodule Google.Protobuf.Timestamp do - @moduledoc """ - A Timestamp represents a point in time independent of any time zone or local - calendar, encoded as a count of seconds and fractions of seconds at - nanosecond resolution. The count is relative to an epoch at UTC midnight on - January 1, 1970, in the proleptic Gregorian calendar which extends the - Gregorian calendar backwards to year one. - - All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap - second table is needed for interpretation, using a [24-hour linear - smear](https://developers.google.com/time/smear). - - The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By - restricting to that range, we ensure that we can convert to and from [RFC - 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. - - ## JSON Mapping - - In JSON format, the Timestamp type is encoded as a string in the - [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the - format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" - where {year} is always expressed using four digits while {month}, {day}, - {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional - seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), - are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone - is required. A proto3 JSON serializer should always use UTC (as indicated by - "Z") when printing the Timestamp type and a proto3 JSON parser should be - able to accept both UTC and other timezones (as indicated by an offset). - - For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past - 01:30 UTC on January 15, 2017. - """ - - use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" - - field :seconds, 1, type: :int64 - field :nanos, 2, type: :int32 - - @doc """ - Converts a `DateTime` struct to a `#{__MODULE__}` struct. - - Note: Elixir `DateTime.from_unix!/2` will convert units to - microseconds internally. Nanosecond precision is not guaranteed. - See examples for details. - - ## Examples - - iex> Timestamp.to_datetime(%Timestamp{seconds: 5, nanos: 0}) - ~U[1970-01-01 00:00:05.000000Z] - - iex> one = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 100}) - ...> two = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 105}) - ...> DateTime.diff(one, two, :nanosecond) - 0 - - """ - @spec to_datetime(__MODULE__.t()) :: DateTime.t() - def to_datetime(%{seconds: seconds, nanos: nanos}) do - DateTime.from_unix!(seconds * 1_000_000_000 + nanos, :nanosecond) - end - - @doc """ - Converts a `#{__MODULE__}` struct to a `DateTime` struct. - - ## Examples - - iex> Timestamp.from_datetime(~U[1970-01-01 00:00:05.000000Z]) - %Timestamp{seconds: 5, nanos: 0} - - """ - @spec from_datetime(DateTime.t()) :: __MODULE__.t() - def from_datetime(%DateTime{} = datetime) do - nanoseconds = DateTime.to_unix(datetime, :nanosecond) - - struct(__MODULE__, %{ - seconds: div(nanoseconds, 1_000_000_000), - nanos: rem(nanoseconds, 1_000_000_000) - }) - end -end diff --git a/lib/google_protos/timestamp.pb.ex b/lib/google_protos/timestamp.pb.ex new file mode 100644 index 0000000..77deb5f --- /dev/null +++ b/lib/google_protos/timestamp.pb.ex @@ -0,0 +1,50 @@ +defmodule Google.Protobuf.Timestamp do + @moduledoc false + + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" + + field :seconds, 1, type: :int64 + field :nanos, 2, type: :int32 + + @doc """ + Converts a `DateTime` struct to a `Google.Protobuf.Timestamp` struct. + + Note: Elixir `DateTime.from_unix!/2` will convert units to + microseconds internally. Nanosecond precision is not guaranteed. + See examples for details. + + ## Examples + + iex> Timestamp.to_datetime(%Timestamp{seconds: 5, nanos: 0}) + ~U[1970-01-01 00:00:05.000000Z] + + iex> one = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 100}) + ...> two = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 105}) + ...> DateTime.diff(one, two, :nanosecond) + 0 + + """ + @spec to_datetime(__MODULE__.t()) :: DateTime.t() + def to_datetime(%{seconds: seconds, nanos: nanos}) do + DateTime.from_unix!(seconds * 1_000_000_000 + nanos, :nanosecond) + end + + @doc """ + Converts a `Google.Protobuf.Timestamp` struct to a `DateTime` struct. + + ## Examples + + iex> Timestamp.from_datetime(~U[1970-01-01 00:00:05.000000Z]) + %Timestamp{seconds: 5, nanos: 0} + + """ + @spec from_datetime(DateTime.t()) :: __MODULE__.t() + def from_datetime(%DateTime{} = datetime) do + nanoseconds = DateTime.to_unix(datetime, :nanosecond) + + struct(__MODULE__, %{ + seconds: div(nanoseconds, 1_000_000_000), + nanos: rem(nanoseconds, 1_000_000_000) + }) + end +end \ No newline at end of file diff --git a/lib/google_protos/wrappers.pb.ex b/lib/google_protos/wrappers.pb.ex index bd6eef2..6934640 100644 --- a/lib/google_protos/wrappers.pb.ex +++ b/lib/google_protos/wrappers.pb.ex @@ -68,4 +68,4 @@ defmodule Google.Protobuf.BytesValue do use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :bytes -end +end \ No newline at end of file diff --git a/priv/templates/README.md b/priv/templates/README.md new file mode 100644 index 0000000..382d2c1 --- /dev/null +++ b/priv/templates/README.md @@ -0,0 +1,9 @@ +# Templates + +These files are placed in the Protobuf generated Elixir modules. For them to match and be inserted: + +1) The template file must match the file in `lib/google_protos`. + +2) The module name must match what module you want the data inserted into. + +3) The template file must include `@moduledoc false` (we remove the first two lines.) diff --git a/priv/templates/duration.pb.ex b/priv/templates/duration.pb.ex new file mode 100644 index 0000000..c7e11a1 --- /dev/null +++ b/priv/templates/duration.pb.ex @@ -0,0 +1,64 @@ +defmodule Google.Protobuf.Duration do + @moduledoc false + + @doc """ + Converts a `Google.Protobuf.Duration` to a value and `System.time_unit()`. + + ## Examples + + iex> Duration.to_time_unit(%Duration{seconds: 10}) + {10, :second} + + iex> Duration.to_time_unit(%Duration{seconds: 20, nanos: 100}) + {20_000_000_100, :nanosecond} + + """ + @spec to_time_unit(__MODULE__.t()) :: {integer(), System.time_unit()} + def to_time_unit(%{seconds: seconds, nanos: 0}) do + {seconds, :second} + end + + def to_time_unit(%{seconds: seconds, nanos: nanos}) do + {seconds * 1_000_000_000 + nanos, :nanosecond} + end + + @doc """ + Converts a value and `System.time_unit()` to a `Google.Protobuf.Duration`. + + ## Examples + + iex> Duration.from_time_unit(11, :second) + %Duration{seconds: 11} + + iex> Duration.from_time_unit(11_111_111, :microsecond) + %Duration{seconds: 11, nanos: 111_111_000} + + """ + @spec from_time_unit(integer(), System.time_unit()) :: __MODULE__.t() + def from_time_unit(seconds, :second) do + struct(__MODULE__, %{ + seconds: seconds + }) + end + + def from_time_unit(millisecond, :millisecond) do + struct(__MODULE__, %{ + seconds: div(millisecond, 1_000), + nanos: rem(millisecond, 1_000) * 1_000_000 + }) + end + + def from_time_unit(millisecond, :microsecond) do + struct(__MODULE__, %{ + seconds: div(millisecond, 1_000_000), + nanos: rem(millisecond, 1_000_000) * 1_000 + }) + end + + def from_time_unit(millisecond, :nanosecond) do + struct(__MODULE__, %{ + seconds: div(millisecond, 1_000_000_000), + nanos: rem(millisecond, 1_000_000_000) + }) + end +end diff --git a/priv/templates/struct.pb.ex b/priv/templates/struct.pb.ex new file mode 100644 index 0000000..0184387 --- /dev/null +++ b/priv/templates/struct.pb.ex @@ -0,0 +1,79 @@ +defmodule Google.Protobuf.Struct do + @moduledoc false + + @doc """ + Converts a `Google.Protobuf.Struct` to a `map()`. + + ## Examples + + iex> Struct.to_map(%Struct{}) + %{} + + """ + @spec to_map(__MODULE__.t()) :: map() + def to_map(struct) do + Map.new(struct.fields, fn {k, v} -> + {k, to_map_value(v)} + end) + end + + defp to_map_value(%{kind: {:null_value, :NULL_VALUE}}), do: nil + defp to_map_value(%{kind: {:number_value, value}}), do: value + defp to_map_value(%{kind: {:string_value, value}}), do: value + defp to_map_value(%{kind: {:bool_value, value}}), do: value + + defp to_map_value(%{kind: {:struct_value, struct}}), + do: to_map(struct) + + defp to_map_value(%{kind: {:list_value, %{values: values}}}), + do: Enum.map(values, &to_map_value/1) + + @doc """ + Converts a `map()` to a `Google.Protobuf.Struct`. + + ## Examples + + iex> Struct.from_map(%{key: "value"}) + %Struct{} + + """ + @spec from_map(map()) :: __MODULE__.t() + def from_map(map) do + struct(__MODULE__, %{ + fields: + Map.new(map, fn {k, v} -> + {to_string(k), from_map_value(v)} + end) + }) + end + + defp from_map_value(nil) do + struct(Google.Protobuf.Value, %{kind: {:null_value, :NULL_VALUE}}) + end + + defp from_map_value(value) when is_number(value) do + struct(Google.Protobuf.Value, %{kind: {:number_value, value}}) + end + + defp from_map_value(value) when is_binary(value) do + struct(Google.Protobuf.Value, %{kind: {:string_value, value}}) + end + + defp from_map_value(value) when is_boolean(value) do + struct(Google.Protobuf.Value, %{kind: {:bool_value, value}}) + end + + defp from_map_value(value) when is_map(value) do + struct(Google.Protobuf.Value, %{kind: {:struct_value, from_map(value)}}) + end + + defp from_map_value(value) when is_list(value) do + struct(Google.Protobuf.Value, %{ + kind: + {:list_value, + struct(Google.Protobuf.ListValue, %{ + values: Enum.map(value, &from_map_value/1) + })} + }) + end +end diff --git a/priv/templates/timestamp.pb.ex b/priv/templates/timestamp.pb.ex new file mode 100644 index 0000000..5887054 --- /dev/null +++ b/priv/templates/timestamp.pb.ex @@ -0,0 +1,45 @@ +defmodule Google.Protobuf.Timestamp do + @moduledoc false + + @doc """ + Converts a `DateTime` struct to a `Google.Protobuf.Timestamp` struct. + + Note: Elixir `DateTime.from_unix!/2` will convert units to + microseconds internally. Nanosecond precision is not guaranteed. + See examples for details. + + ## Examples + + iex> Timestamp.to_datetime(%Timestamp{seconds: 5, nanos: 0}) + ~U[1970-01-01 00:00:05.000000Z] + + iex> one = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 100}) + ...> two = Timestamp.to_datetime(%Timestamp{seconds: 10, nanos: 105}) + ...> DateTime.diff(one, two, :nanosecond) + 0 + + """ + @spec to_datetime(__MODULE__.t()) :: DateTime.t() + def to_datetime(%{seconds: seconds, nanos: nanos}) do + DateTime.from_unix!(seconds * 1_000_000_000 + nanos, :nanosecond) + end + + @doc """ + Converts a `Google.Protobuf.Timestamp` struct to a `DateTime` struct. + + ## Examples + + iex> Timestamp.from_datetime(~U[1970-01-01 00:00:05.000000Z]) + %Timestamp{seconds: 5, nanos: 0} + + """ + @spec from_datetime(DateTime.t()) :: __MODULE__.t() + def from_datetime(%DateTime{} = datetime) do + nanoseconds = DateTime.to_unix(datetime, :nanosecond) + + struct(__MODULE__, %{ + seconds: div(nanoseconds, 1_000_000_000), + nanos: rem(nanoseconds, 1_000_000_000) + }) + end +end From aecbeacae1c1562f8ee27e04f250e3570807a298 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Fri, 16 Jun 2023 10:55:41 -0600 Subject: [PATCH 5/7] run mix format after generate script --- generate_google_protos.sh | 2 ++ lib/google_protos/any.pb.ex | 2 +- lib/google_protos/duration.pb.ex | 2 +- lib/google_protos/empty.pb.ex | 2 +- lib/google_protos/field_mask.pb.ex | 2 +- lib/google_protos/struct.pb.ex | 2 +- lib/google_protos/timestamp.pb.ex | 2 +- lib/google_protos/wrappers.pb.ex | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/generate_google_protos.sh b/generate_google_protos.sh index 46b6c7d..f653d77 100755 --- a/generate_google_protos.sh +++ b/generate_google_protos.sh @@ -31,3 +31,5 @@ for template_file in priv/templates/*.pb.ex; do done fi done + +mix format diff --git a/lib/google_protos/any.pb.ex b/lib/google_protos/any.pb.ex index 3d6c939..7472c29 100644 --- a/lib/google_protos/any.pb.ex +++ b/lib/google_protos/any.pb.ex @@ -5,4 +5,4 @@ defmodule Google.Protobuf.Any do field :type_url, 1, type: :string, json_name: "typeUrl" field :value, 2, type: :bytes -end \ No newline at end of file +end diff --git a/lib/google_protos/duration.pb.ex b/lib/google_protos/duration.pb.ex index 96d1cc2..ad98262 100644 --- a/lib/google_protos/duration.pb.ex +++ b/lib/google_protos/duration.pb.ex @@ -66,4 +66,4 @@ defmodule Google.Protobuf.Duration do nanos: rem(millisecond, 1_000_000_000) }) end -end \ No newline at end of file +end diff --git a/lib/google_protos/empty.pb.ex b/lib/google_protos/empty.pb.ex index ded0331..334a639 100644 --- a/lib/google_protos/empty.pb.ex +++ b/lib/google_protos/empty.pb.ex @@ -2,4 +2,4 @@ defmodule Google.Protobuf.Empty do @moduledoc false use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" -end \ No newline at end of file +end diff --git a/lib/google_protos/field_mask.pb.ex b/lib/google_protos/field_mask.pb.ex index d64b861..6950164 100644 --- a/lib/google_protos/field_mask.pb.ex +++ b/lib/google_protos/field_mask.pb.ex @@ -4,4 +4,4 @@ defmodule Google.Protobuf.FieldMask do use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :paths, 1, repeated: true, type: :string -end \ No newline at end of file +end diff --git a/lib/google_protos/struct.pb.ex b/lib/google_protos/struct.pb.ex index d96ae9b..3835e1e 100644 --- a/lib/google_protos/struct.pb.ex +++ b/lib/google_protos/struct.pb.ex @@ -125,4 +125,4 @@ defmodule Google.Protobuf.ListValue do use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :values, 1, repeated: true, type: Google.Protobuf.Value -end \ No newline at end of file +end diff --git a/lib/google_protos/timestamp.pb.ex b/lib/google_protos/timestamp.pb.ex index 77deb5f..9ef9d82 100644 --- a/lib/google_protos/timestamp.pb.ex +++ b/lib/google_protos/timestamp.pb.ex @@ -47,4 +47,4 @@ defmodule Google.Protobuf.Timestamp do nanos: rem(nanoseconds, 1_000_000_000) }) end -end \ No newline at end of file +end diff --git a/lib/google_protos/wrappers.pb.ex b/lib/google_protos/wrappers.pb.ex index 6934640..bd6eef2 100644 --- a/lib/google_protos/wrappers.pb.ex +++ b/lib/google_protos/wrappers.pb.ex @@ -68,4 +68,4 @@ defmodule Google.Protobuf.BytesValue do use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.12.0" field :value, 1, type: :bytes -end \ No newline at end of file +end From 016b406f7fea4fef623e4691f17b073177a79de4 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Fri, 16 Jun 2023 11:09:29 -0600 Subject: [PATCH 6/7] make compatible with macOS --- generate_google_protos.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generate_google_protos.sh b/generate_google_protos.sh index f653d77..f434d2d 100755 --- a/generate_google_protos.sh +++ b/generate_google_protos.sh @@ -21,13 +21,13 @@ for template_file in priv/templates/*.pb.ex; do replacement_file="lib/google_protos/$file_name" if [ -f "$replacement_file" ]; then - for module in $(sed -n -E "s/defmodule\s(.+)\sdo/\1/p" "$template_file"); do - replacement_text=$(cat "$template_file" | sed -n "/^defmodule\s$module\sdo$/,/^end$/p" | sed 1,2d) - line_to_replace=$(cat -n "$replacement_file" | sed -n "/defmodule\s$module\sdo$/,/end$/p" | grep "end" | awk '{print $1}') + for module in $(sed -n -E "s/defmodule (.+) do/\1/p" "$template_file"); do + replacement_text=$(cat "$template_file" | sed -n "/^defmodule $module do$/,/^end$/p" | sed 1,2d) + line_to_replace=$(cat -n "$replacement_file" | sed -n "/defmodule $module do$/,/end$/p" | grep "end" | awk '{print $1}') export REPLACEMENT_TEXT=$replacement_text perl -i -pe "s/end/\$ENV{\"REPLACEMENT_TEXT\"}/g if $. == $line_to_replace" "$replacement_file" - unset $REPLACEMENT_TEXT + unset REPLACEMENT_TEXT done fi done From ea92b2fff1fea8c7efd25e739f782fa5b5e54df4 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Fri, 16 Jun 2023 11:11:02 -0600 Subject: [PATCH 7/7] unmodify .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0798449..d40d812 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ erl_crash.dump # Ignore package tarball (built via "mix hex.build"). google_protobuf-*.tar +