From 56889c0fdcbc94b27c3a0973d3d4a0be4bc0015a Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 9 Jul 2024 15:13:32 +0200 Subject: [PATCH] json array implicits for doobie --- .../integration/V1.2.15__actors_json_seq.ddl | 21 ++++ .../V1.2.16__insert_actors_json.sql | 23 +++++ .../V1.2.17__retrieve_actors_json.sql | 22 +++++ .../V1.2.18__retrieve_actors_jsonb.sql | 22 +++++ .../doobie/postgres/implicits/package.scala | 96 +++++++++++++++++++ .../absa/db/fadb/doobie/JsonArrayTests.scala | 75 +++++++++++++++ .../db/fadb/testing/classes/DoobieTest.scala | 6 ++ project/Dependencies.scala | 16 +++- 8 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl create mode 100644 demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql create mode 100644 demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql create mode 100644 demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql create mode 100644 doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala create mode 100644 doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala diff --git a/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl b/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl new file mode 100644 index 00000000..de4c6c91 --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl @@ -0,0 +1,21 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS integration.actors_json_seq ( + id SERIAL PRIMARY KEY, + actors_json JSON[], + actors_jsonb JSONB[] +); diff --git a/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql b/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql new file mode 100644 index 00000000..b59c5ab7 --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql @@ -0,0 +1,23 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.insert_actors_json(actorsJson JSON[], actorsJsonb JSONB[]) + RETURNS void AS $$ +BEGIN + INSERT INTO integration.actors_json_seq (actors_json, actors_jsonb) + VALUES (actorsJson, actorsJsonb); +END; +$$ LANGUAGE plpgsql; diff --git a/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql b/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql new file mode 100644 index 00000000..a34a472a --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql @@ -0,0 +1,22 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.retrieve_actors_json(idUntil INT) + RETURNS TABLE(actors_json JSON[]) AS $$ +BEGIN + RETURN QUERY SELECT a.actors_json FROM integration.actors_json_seq AS a WHERE id <= idUntil; +END; +$$ LANGUAGE plpgsql; diff --git a/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql b/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql new file mode 100644 index 00000000..bb5e3e4f --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql @@ -0,0 +1,22 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.retrieve_actors_jsonb(idUntil INT) + RETURNS TABLE(actors_jsonb JSONB[]) AS $$ +BEGIN + RETURN QUERY SELECT a.actors_jsonb FROM integration.actors_json_seq AS a WHERE id <= idUntil; +END; +$$ LANGUAGE plpgsql; diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala new file mode 100644 index 00000000..625d1d02 --- /dev/null +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/implicits/package.scala @@ -0,0 +1,96 @@ +package za.co.absa.db.fadb.doobie.postgres + +import cats.Show +import cats.data.NonEmptyList +import doobie.postgres.implicits._ +import doobie.{Get, Put} +import io.circe.{Json => CirceJson} +import org.postgresql.jdbc.PgArray +import org.postgresql.util.PGobject +import io.circe.parser._ + +import scala.util.Try + +package object implicits { + + private implicit val showPgArray: Show[PgArray] = Show.fromToString + + implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] + .tmap(map => map.map { case (k, v) => k -> Option(v) }) + + private def circeJsonListToPGJsonArrayString(jsonList: List[CirceJson]): String = { + val arrayElements = jsonList.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s""""$escapedJsonString"""" + } + + arrayElements.mkString("{", ",", "}") + } + + private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[CirceJson]] = { + Try { + Option(pgArray.getArray) match { + case Some(array: Array[_]) => array.collect { + case str: String => parse(str).toTry.get + case other => parse(other.toString).toTry.get + }.toList + case None => List.empty[CirceJson] + case _ => throw new IllegalArgumentException("Unexpected type encountered.") + } + } + .toEither + .left.map(_.getMessage) + } + + object Sequence { + + implicit val get: Get[Seq[String]] = Get[List[String]].map(_.toSeq) + implicit val put: Put[Seq[String]] = Put[List[String]].contramap(_.toList) + + } + + object Json { + + implicit val jsonArrayPut: Put[List[CirceJson]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("json[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("json[]") + o.setValue(circeJsonListToPGJsonArrayString(a)) + o + } + } + + implicit val jsonArrayGet: Get[List[CirceJson]] = { + Get.Advanced + .other[PgArray]( + NonEmptyList.of("json[]") + ) + .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) + } + + } + + object Jsonb { + + implicit val jsonbArrayPut: Put[List[CirceJson]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("jsonb[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("jsonb[]") + o.setValue(circeJsonListToPGJsonArrayString(a)) + o + } + } + + } + +} diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala new file mode 100644 index 00000000..1ff37542 --- /dev/null +++ b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayTests.scala @@ -0,0 +1,75 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.db.fadb.doobie + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import doobie.implicits.toSqlInterpolator +import io.circe.Json +import io.circe.syntax.EncoderOps +import org.scalatest.funsuite.AnyFunSuite +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieFunction.{DoobieMultipleResultFunction, DoobieSingleResultFunction} +import za.co.absa.db.fadb.testing.classes.DoobieTest + +import za.co.absa.db.fadb.doobie.postgres.implicits.Json.jsonArrayGet + +class JsonArrayTests extends AnyFunSuite with DoobieTest { + + class InsertActorsJson(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieSingleResultFunction[List[Actor], Unit, IO] ( + values => { + val actorsAsJsonList = values.map(_.asJson) + Seq( + { + import za.co.absa.db.fadb.doobie.postgres.implicits.Json.jsonArrayPut + fr"$actorsAsJsonList" + }, + { + import za.co.absa.db.fadb.doobie.postgres.implicits.Jsonb.jsonbArrayPut + fr"$actorsAsJsonList" + } + ) + } + ) + + class RetrieveActorsJson(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieMultipleResultFunction[Int, List[Json], IO] ( + values => Seq(fr"$values") + ) + + class RetrieveActorsJsonb(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieMultipleResultFunction[Int, List[Json], IO] ( + values => Seq(fr"$values") + ) + + private val insertActorsJson = new InsertActorsJson()(Integration, new DoobieEngine(transactor)) + + test("Retrieve Actors from json[] and jsonb[] columns"){ + val expectedActors = List(Actor(1, "John", "Doe"), Actor(2, "Jane", "Doe")) + insertActorsJson(expectedActors).unsafeRunSync() + + val retrieveActorsJson = new RetrieveActorsJson()(Integration, new DoobieEngine(transactor)) + val actualActorsJson = retrieveActorsJson(2).unsafeRunSync() + assert(expectedActors == actualActorsJson.head.map(_.as[Actor]).map(_.toTry.get)) + + val retrieveActorsJsonb = new RetrieveActorsJsonb()(Integration, new DoobieEngine(transactor)) + val actualActorsJsonb = retrieveActorsJsonb(2).unsafeRunSync() + assert(expectedActors == actualActorsJsonb.head.map(_.as[Actor]).map(_.toTry.get)) + } + +} diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala b/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala index e6cbf5ef..fa570b53 100644 --- a/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala +++ b/doobie/src/test/scala/za/co/absa/db/fadb/testing/classes/DoobieTest.scala @@ -20,9 +20,15 @@ import cats.effect.IO import doobie.util.transactor.Transactor import doobie.util.transactor.Transactor.Aux import za.co.absa.db.fadb.DBSchema +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto._ trait DoobieTest { case class Actor(actorId: Int, firstName: String, lastName: String) + object Actor { + implicit val actorEncoder: Encoder[Actor] = deriveEncoder + implicit val actorDecoder: Decoder[Actor] = deriveDecoder + } case class GetActorsQueryParameters(firstName: Option[String], lastName: Option[String]) case class GetActorsByLastnameQueryParameters(lastName: String, firstName: Option[String] = None) case class CreateActorRequestBody(firstName: String, lastName: String) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 256fc732..103a8f71 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -import sbt._ +import sbt.* object Dependencies { @@ -44,7 +44,7 @@ object Dependencies { } def doobieDependencies(scalaVersion: String): Seq[ModuleID] = { - commonDependencies(scalaVersion) ++ Seq( + commonDependencies(scalaVersion) ++ jsonSerdeDependencies ++ Seq( "org.tpolecat" %% "doobie-core" % "1.0.0-RC2", "org.tpolecat" %% "doobie-hikari" % "1.0.0-RC2", "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2" @@ -56,4 +56,16 @@ object Dependencies { Seq(postgresql) } + + private def jsonSerdeDependencies: Seq[ModuleID] = { + lazy val circeCore = "io.circe" %% "circe-core" % "0.14.7" + lazy val circeParser = "io.circe" %% "circe-parser" % "0.14.7" + lazy val circeGeneric = "io.circe" %% "circe-generic" % "0.14.7" + + Seq( + circeCore, + circeParser, + circeGeneric, + ) + } }