From 24d4930ae803731db2875549c105b16ad68a9fb5 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Mon, 12 Jun 2017 12:56:30 +0200 Subject: [PATCH 01/61] Add search dsl skeleton --- .../api/app/foxcomm/search/api/Api.scala | 2 +- .../app/foxcomm/search/SearchService.scala | 12 ++-- .../core/app/foxcomm/search/dsl.scala | 59 +++++++++++++++++++ .../core/app/foxcomm/search/package.scala | 7 +++ .../core/app/foxcomm/search/payload.scala | 9 ++- search-service/project/Dependencies.scala | 6 +- 6 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 search-service/core/app/foxcomm/search/dsl.scala create mode 100644 search-service/core/app/foxcomm/search/package.scala diff --git a/search-service/api/app/foxcomm/search/api/Api.scala b/search-service/api/app/foxcomm/search/api/Api.scala index c9d57f575a..097d5969fb 100644 --- a/search-service/api/app/foxcomm/search/api/Api.scala +++ b/search-service/api/app/foxcomm/search/api/Api.scala @@ -6,7 +6,7 @@ import com.twitter.finagle.http.Status import com.twitter.util.Await import foxcomm.search._ import foxcomm.utils.finch._ -import io.circe.generic.auto._ +import io.circe.generic.extras.auto._ import io.finch._ import io.finch.circe._ import org.elasticsearch.common.ValidationException diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index 4a1bbbd11f..fadcf05f37 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -16,12 +16,14 @@ class SearchService(private val client: ElasticClient) extends AnyVal { searchQuery: SearchQuery, searchSize: Int, searchFrom: Option[Int])(implicit ec: ExecutionContext): Future[SearchResult] = { - val baseQuery = search in searchIndex size searchSize rawQuery Json - .fromJsonObject(searchQuery.query) - .noSpaces - val query = searchQuery.fields.fold(baseQuery)(fields ⇒ baseQuery sourceInclude (fields.toList: _*)) + val withQuery = searchQuery match { + case SearchQuery.ES(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces + case SearchQuery.FC(query, _) ⇒ ??? // TODO + } + val baseSearch = withQuery(search in searchIndex size searchSize) + val limitedSearch = searchQuery.fields.fold(baseSearch)(fields ⇒ baseSearch sourceInclude (fields.toList: _*)) client - .execute(searchFrom.fold(query)(query from)) + .execute(searchFrom.fold(limitedSearch)(limitedSearch from)) .map(response ⇒ SearchResult(result = response.hits.collect { case ExtractJsonObject(obj) ⇒ obj diff --git a/search-service/core/app/foxcomm/search/dsl.scala b/search-service/core/app/foxcomm/search/dsl.scala new file mode 100644 index 0000000000..ba56785ec6 --- /dev/null +++ b/search-service/core/app/foxcomm/search/dsl.scala @@ -0,0 +1,59 @@ +package foxcomm.search.dsl + +import cats.data.NonEmptyList +import io.circe.Decoder.Result +import io.circe.{Decoder, HCursor} +import scala.collection.immutable + +final case class FCQuery(query: NonEmptyList[QueryFunction]) +object FCQuery { + implicit val decoder: Decoder[FCQuery] = new Decoder[FCQuery] { + def apply(c: HCursor): Result[FCQuery] = ??? + } +} + +sealed trait QueryField +object QueryField { + final case class Single(field: String) extends QueryField + final case class Multiple(fields: NonEmptyList[String]) extends QueryField + final case class Compound(field: String) extends QueryField +} + +sealed trait QueryValue[T] +object QueryValue { + final case class Single[T](value: T) extends QueryValue[T] + final case class Multiple[T](values: NonEmptyList[T]) extends QueryValue[T] + final case class Map[T](map: immutable.Map[RangeFunction, T]) extends QueryValue[T] +} + +sealed trait RangeFunction +object RangeFunction { + case object Lt extends RangeFunction + case object Lte extends RangeFunction + case object Gt extends RangeFunction + case object Gte extends RangeFunction + case object Eq extends RangeFunction + case object Neq extends RangeFunction +} + +sealed trait QueryFunction { + type Value + + def value: QueryValue[Value] +} +object QueryFunction { + sealed trait FilterContext[T] extends QueryFunction { + type Value = T + } + sealed trait QueryContext[T] extends QueryFunction { + type Value = T + } + sealed trait FieldContext { + def in: QueryField + } + final case class Contains[T](in: QueryField, value: QueryValue[T]) extends QueryContext[T] with FieldContext + final case class Matches[T](in: QueryField, value: QueryValue[T]) extends QueryContext[T] with FieldContext + final case class Range[T](in: QueryField, value: QueryValue.Map[T]) extends FilterContext[T] with FieldContext + final case class Is[T](in: QueryField, value: QueryValue[T]) extends FilterContext[T] with FieldContext + final case class State(value: QueryValue.Single[String]) extends FilterContext[String] +} diff --git a/search-service/core/app/foxcomm/search/package.scala b/search-service/core/app/foxcomm/search/package.scala new file mode 100644 index 0000000000..4a2038472f --- /dev/null +++ b/search-service/core/app/foxcomm/search/package.scala @@ -0,0 +1,7 @@ +package foxcomm + +import io.circe.generic.extras.Configuration + +package object search { + implicit val configuration: Configuration = Configuration.default.withDiscriminator("type") +} diff --git a/search-service/core/app/foxcomm/search/payload.scala b/search-service/core/app/foxcomm/search/payload.scala index a6952a7166..3492253217 100644 --- a/search-service/core/app/foxcomm/search/payload.scala +++ b/search-service/core/app/foxcomm/search/payload.scala @@ -1,6 +1,13 @@ package foxcomm.search import cats.data.NonEmptyList +import foxcomm.search.dsl.FCQuery import io.circe.JsonObject -final case class SearchQuery(query: JsonObject, fields: Option[NonEmptyList[String]]) +sealed trait SearchQuery { + def fields: Option[NonEmptyList[String]] +} +object SearchQuery { + final case class ES(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchQuery + final case class FC(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchQuery +} diff --git a/search-service/project/Dependencies.scala b/search-service/project/Dependencies.scala index 9714d4f4a7..7d32fd03c7 100644 --- a/search-service/project/Dependencies.scala +++ b/search-service/project/Dependencies.scala @@ -20,9 +20,9 @@ object Dependencies { ) val circe = Seq( - "io.circe" %% "circe-core" % versions.circe, - "io.circe" %% "circe-generic" % versions.circe, - "io.circe" %% "circe-parser" % versions.circe + "io.circe" %% "circe-core" % versions.circe, + "io.circe" %% "circe-generic-extras" % versions.circe, + "io.circe" %% "circe-parser" % versions.circe ) val finch = Seq( From 5ad133f73d4c0dd5e130266c32acc4f011ed546a Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Mon, 12 Jun 2017 20:13:21 +0200 Subject: [PATCH 02/61] Add initial implementation --- .../app/foxcomm/search/SearchService.scala | 12 +- .../core/app/foxcomm/search/dsl.scala | 155 ++++++++++++++---- .../core/app/foxcomm/search/package.scala | 3 +- .../core/app/foxcomm/search/payload.scala | 3 +- 4 files changed, 132 insertions(+), 41 deletions(-) diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index fadcf05f37..13d59eb446 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -18,10 +18,18 @@ class SearchService(private val client: ElasticClient) extends AnyVal { searchFrom: Option[Int])(implicit ec: ExecutionContext): Future[SearchResult] = { val withQuery = searchQuery match { case SearchQuery.ES(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces - case SearchQuery.FC(query, _) ⇒ ??? // TODO + case SearchQuery.FC(query, _) ⇒ + (_: SearchDefinition) bool { + query.query.foldLeft(new BoolQueryDefinition) { + case (bool, QueryFunction.Is(in, value)) ⇒ + bool.filter(in.toList.map(termsQuery(_, value.fold(QueryFunction.listOfAnyValueF): _*))) + case (bool, _) ⇒ bool // TODO: implement rest of cases + } + } } val baseSearch = withQuery(search in searchIndex size searchSize) - val limitedSearch = searchQuery.fields.fold(baseSearch)(fields ⇒ baseSearch sourceInclude (fields.toList: _*)) + val limitedSearch = + searchQuery.fields.fold(baseSearch)(fields ⇒ baseSearch sourceInclude (fields.toList: _*)) client .execute(searchFrom.fold(limitedSearch)(limitedSearch from)) .map(response ⇒ diff --git a/search-service/core/app/foxcomm/search/dsl.scala b/search-service/core/app/foxcomm/search/dsl.scala index ba56785ec6..36ae121cf6 100644 --- a/search-service/core/app/foxcomm/search/dsl.scala +++ b/search-service/core/app/foxcomm/search/dsl.scala @@ -1,59 +1,142 @@ -package foxcomm.search.dsl +package foxcomm.search import cats.data.NonEmptyList +import cats.syntax.either._ import io.circe.Decoder.Result -import io.circe.{Decoder, HCursor} +import io.circe.generic.extras.auto._ +import io.circe.{Decoder, DecodingFailure, HCursor} import scala.collection.immutable +import shapeless._ -final case class FCQuery(query: NonEmptyList[QueryFunction]) -object FCQuery { - implicit val decoder: Decoder[FCQuery] = new Decoder[FCQuery] { - def apply(c: HCursor): Result[FCQuery] = ??? - } +sealed trait QueryField extends Product with Serializable { + def toList: List[String] } - -sealed trait QueryField object QueryField { - final case class Single(field: String) extends QueryField - final case class Multiple(fields: NonEmptyList[String]) extends QueryField - final case class Compound(field: String) extends QueryField -} - -sealed trait QueryValue[T] -object QueryValue { - final case class Single[T](value: T) extends QueryValue[T] - final case class Multiple[T](values: NonEmptyList[T]) extends QueryValue[T] - final case class Map[T](map: immutable.Map[RangeFunction, T]) extends QueryValue[T] + final case class Single(field: String) extends QueryField { + def toList: List[String] = List(field) + } + final case class Multiple(fields: NonEmptyList[String]) extends QueryField { + def toList: List[String] = fields.toList + } + final case class Compound(field: String) extends QueryField { + def toList: List[String] = ??? // TODO: implement compound fields like $text + } } sealed trait RangeFunction object RangeFunction { - case object Lt extends RangeFunction + case object Lt extends RangeFunction case object Lte extends RangeFunction - case object Gt extends RangeFunction + case object Gt extends RangeFunction case object Gte extends RangeFunction - case object Eq extends RangeFunction + case object Eq extends RangeFunction case object Neq extends RangeFunction } -sealed trait QueryFunction { - type Value +sealed trait QueryFunction extends Product with Serializable { + type Value <: Coproduct - def value: QueryValue[Value] + def value: Value } +@SuppressWarnings( + Array("org.wartremover.warts.AsInstanceOf", + "org.wartremover.warts.Equals", + "org.wartremover.warts.ExplicitImplicitTypes")) object QueryFunction { - sealed trait FilterContext[T] extends QueryFunction { - type Value = T + type QueryValue[T] = T :+: NonEmptyList[T] :+: CNil + // TODO: what with precision loss? current elastic4s version operates on Double's in range query anyway + type SingleValue = Double :+: String :+: CNil + type CompoundValue = QueryValue[SingleValue] + type RangeValue = immutable.Map[RangeFunction, SingleValue] :+: CNil + type StateValue = allW.T :+: activeW.T :+: inactiveW.T :+: CNil + + val allW = Witness("all") + val activeW = Witness("active") + val inactiveW = Witness("inactive") + + object queryFieldF extends Poly1 { + implicit def caseField = at[String](QueryField.Single) + implicit def caseFields = at[NonEmptyList[String]](QueryField.Multiple) + } + + // we autobox `Double` here via type cast + object listOfAnyValueF extends Poly1 { + implicit def caseValue = at[SingleValue](v ⇒ List(v.unify.asInstanceOf[AnyRef])) + implicit def caseValues = at[NonEmptyList[SingleValue]](_.map(_.unify).toList.asInstanceOf[List[AnyRef]]) + } + + implicit def decodeQueryValue[T: Decoder]: Decoder[QueryValue[T]] = + Decoder[T] + .map(Coproduct[QueryValue[T]](_)) + .or(Decoder.decodeNonEmptyList[T].map(Coproduct[QueryValue[T]](_))) + + implicit val decodeSingleValue: Decoder[SingleValue] = + Decoder.decodeDouble.map(Inl(_)).or(Decoder.decodeString.map(s ⇒ Inr(Inl(s)))) + + implicit val decodeState: Decoder[StateValue] = + Decoder.decodeString.emap { + case v if v == allW.value ⇒ Either.right(Coproduct[StateValue](allW.value)) + case v if v == activeW.value ⇒ Either.right(Coproduct[StateValue](activeW.value)) + case v if v == inactiveW.value ⇒ Either.right(Coproduct[StateValue](inactiveW.value)) + case _ ⇒ Either.left("unknown state defined") + } + + implicit val decodeQueryField: Decoder[QueryField] = new Decoder[QueryField] { + def apply(c: HCursor): Result[QueryField] = + c.downField("in").as[QueryValue[String]].right.map(_.fold(queryFieldF)) + } + + implicit val decodeCompoundValue: Decoder[CompoundValue] = decodeQueryValue[SingleValue] + + implicit val decodeRange: Decoder[RangeValue] = new Decoder[RangeValue] { + def apply(c: HCursor): Result[RangeValue] = ??? // TODO: implement decoder + } + + private[this] val decoderMap = Map( + "contains" → Decoder[Contains], + "matches" → Decoder[Matches], + "is" → Decoder[Is], + "range" → Decoder[Range], + "state" → Decoder[State] + ) + + implicit val decodeQueryFunction: Decoder[QueryFunction] = new Decoder[QueryFunction] { + def apply(c: HCursor): Result[QueryFunction] = + for { + tpe ← c.downField("type").as[String] + decoder ← decoderMap + .get(tpe) + .map(Either.right(_)) + .getOrElse(Either.left(DecodingFailure("", c.history))) + value ← decoder(c) + } yield value } - sealed trait QueryContext[T] extends QueryFunction { - type Value = T + + final case class Contains(in: QueryField, value: CompoundValue) extends QueryFunction { + type Value = CompoundValue + } + final case class Matches(in: QueryField, value: CompoundValue) extends QueryFunction { + type Value = CompoundValue + } + final case class Range(in: QueryField.Single, value: RangeValue) extends QueryFunction { + type Value = RangeValue + } + final case class Is(in: QueryField, value: CompoundValue) extends QueryFunction { + type Value = CompoundValue } - sealed trait FieldContext { - def in: QueryField + final case class State(value: StateValue) extends QueryFunction { + type Value = StateValue } - final case class Contains[T](in: QueryField, value: QueryValue[T]) extends QueryContext[T] with FieldContext - final case class Matches[T](in: QueryField, value: QueryValue[T]) extends QueryContext[T] with FieldContext - final case class Range[T](in: QueryField, value: QueryValue.Map[T]) extends FilterContext[T] with FieldContext - final case class Is[T](in: QueryField, value: QueryValue[T]) extends FilterContext[T] with FieldContext - final case class State(value: QueryValue.Single[String]) extends FilterContext[String] +} + +final case class FCQuery(query: NonEmptyList[QueryFunction]) +object FCQuery { + // TODO: check performance (e.g. no doubled attempt for query function decoding) if query is malformed + // should be fast, as it'd either fail fast on lack of array + // or on first (or single) malformed query function + implicit val decodeFCQuery: Decoder[FCQuery] = + Decoder + .decodeNonEmptyList[QueryFunction] + .or(Decoder[QueryFunction].map(NonEmptyList.of(_))) + .map(FCQuery(_)) } diff --git a/search-service/core/app/foxcomm/search/package.scala b/search-service/core/app/foxcomm/search/package.scala index 4a2038472f..10e39f385e 100644 --- a/search-service/core/app/foxcomm/search/package.scala +++ b/search-service/core/app/foxcomm/search/package.scala @@ -3,5 +3,6 @@ package foxcomm import io.circe.generic.extras.Configuration package object search { - implicit val configuration: Configuration = Configuration.default.withDiscriminator("type") + implicit val configuration: Configuration = + Configuration.default.withDiscriminator("type").withSnakeCaseKeys } diff --git a/search-service/core/app/foxcomm/search/payload.scala b/search-service/core/app/foxcomm/search/payload.scala index 3492253217..63f08437f8 100644 --- a/search-service/core/app/foxcomm/search/payload.scala +++ b/search-service/core/app/foxcomm/search/payload.scala @@ -1,7 +1,6 @@ package foxcomm.search import cats.data.NonEmptyList -import foxcomm.search.dsl.FCQuery import io.circe.JsonObject sealed trait SearchQuery { @@ -9,5 +8,5 @@ sealed trait SearchQuery { } object SearchQuery { final case class ES(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchQuery - final case class FC(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchQuery + final case class FC(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchQuery } From 8a36edaa8cca44c20a29bf21c9596a3965aad6e3 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 10:13:27 +0200 Subject: [PATCH 03/61] Implement range Add basic range implementation. Do some fixes. --- .../app/foxcomm/search/SearchService.scala | 6 +- .../core/app/foxcomm/search/dsl.scala | 151 +++++++----------- .../core/app/foxcomm/search/payload.scala | 4 +- 3 files changed, 63 insertions(+), 98 deletions(-) diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index 13d59eb446..0fa6a37cbd 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -17,11 +17,11 @@ class SearchService(private val client: ElasticClient) extends AnyVal { searchSize: Int, searchFrom: Option[Int])(implicit ec: ExecutionContext): Future[SearchResult] = { val withQuery = searchQuery match { - case SearchQuery.ES(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces - case SearchQuery.FC(query, _) ⇒ + case SearchQuery.es(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces + case SearchQuery.fc(query, _) ⇒ (_: SearchDefinition) bool { query.query.foldLeft(new BoolQueryDefinition) { - case (bool, QueryFunction.Is(in, value)) ⇒ + case (bool, QueryFunction.is(in, value)) ⇒ bool.filter(in.toList.map(termsQuery(_, value.fold(QueryFunction.listOfAnyValueF): _*))) case (bool, _) ⇒ bool // TODO: implement rest of cases } diff --git a/search-service/core/app/foxcomm/search/dsl.scala b/search-service/core/app/foxcomm/search/dsl.scala index 36ae121cf6..48689e63bf 100644 --- a/search-service/core/app/foxcomm/search/dsl.scala +++ b/search-service/core/app/foxcomm/search/dsl.scala @@ -2,25 +2,37 @@ package foxcomm.search import cats.data.NonEmptyList import cats.syntax.either._ -import io.circe.Decoder.Result -import io.circe.generic.extras.auto._ -import io.circe.{Decoder, DecodingFailure, HCursor} -import scala.collection.immutable +import io.circe.generic.extras.semiauto._ +import io.circe.{Decoder, JsonNumber, KeyDecoder} import shapeless._ -sealed trait QueryField extends Product with Serializable { +sealed trait QueryField { def toList: List[String] } object QueryField { final case class Single(field: String) extends QueryField { def toList: List[String] = List(field) } + object Single { + implicit val decodeSingle: Decoder[Single] = Decoder.decodeString + .emap { + case s if s.startsWith("$") ⇒ Either.left(s"Defined unknown special query field $s") + case s ⇒ Either.right(s) + } + .map(Single(_)) + } + final case class Multiple(fields: NonEmptyList[String]) extends QueryField { def toList: List[String] = fields.toList } - final case class Compound(field: String) extends QueryField { - def toList: List[String] = ??? // TODO: implement compound fields like $text + object Multiple { + implicit val decodeMultiple: Decoder[Multiple] = + Decoder.decodeNonEmptyList[String].map(Multiple(_)) } + + implicit val decodeQueryField: Decoder[QueryField] = + Decoder[Single].map(s ⇒ s: QueryField) or + Decoder[Multiple].map(m ⇒ m: QueryField) } sealed trait RangeFunction @@ -29,111 +41,64 @@ object RangeFunction { case object Lte extends RangeFunction case object Gt extends RangeFunction case object Gte extends RangeFunction - case object Eq extends RangeFunction - case object Neq extends RangeFunction + + implicit val decodeRangeFunction: KeyDecoder[RangeFunction] = KeyDecoder.instance { + case "lt" | "<" ⇒ Some(Lt) + case "lte" | "<=" ⇒ Some(Lte) + case "gt" | ">" ⇒ Some(Gt) + case "gte" | ">=" ⇒ Some(Gte) + } } -sealed trait QueryFunction extends Product with Serializable { - type Value <: Coproduct +sealed trait EntityState +object EntityState { + case object all extends EntityState + case object active extends EntityState + case object inactive extends EntityState - def value: Value + implicit val decodeEntityState: Decoder[EntityState] = deriveEnumerationDecoder[EntityState] } -@SuppressWarnings( - Array("org.wartremover.warts.AsInstanceOf", - "org.wartremover.warts.Equals", - "org.wartremover.warts.ExplicitImplicitTypes")) + +sealed trait QueryFunction +@SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.ExplicitImplicitTypes")) object QueryFunction { - type QueryValue[T] = T :+: NonEmptyList[T] :+: CNil - // TODO: what with precision loss? current elastic4s version operates on Double's in range query anyway - type SingleValue = Double :+: String :+: CNil - type CompoundValue = QueryValue[SingleValue] - type RangeValue = immutable.Map[RangeFunction, SingleValue] :+: CNil - type StateValue = allW.T :+: activeW.T :+: inactiveW.T :+: CNil - - val allW = Witness("all") - val activeW = Witness("active") - val inactiveW = Witness("inactive") - - object queryFieldF extends Poly1 { - implicit def caseField = at[String](QueryField.Single) - implicit def caseFields = at[NonEmptyList[String]](QueryField.Multiple) - } + type SingleValue = JsonNumber :+: String :+: CNil + type MultipleValues = NonEmptyList[JsonNumber] :+: NonEmptyList[String] :+: CNil + type CompoundValue = SingleValue :+: MultipleValues :+: CNil + type RangeValue = Map[RangeFunction, JsonNumber] :+: Map[RangeFunction, String] :+: CNil - // we autobox `Double` here via type cast object listOfAnyValueF extends Poly1 { implicit def caseValue = at[SingleValue](v ⇒ List(v.unify.asInstanceOf[AnyRef])) - implicit def caseValues = at[NonEmptyList[SingleValue]](_.map(_.unify).toList.asInstanceOf[List[AnyRef]]) + implicit def caseValues = at[MultipleValues](_.unify.toList.asInstanceOf[List[AnyRef]]) } - implicit def decodeQueryValue[T: Decoder]: Decoder[QueryValue[T]] = - Decoder[T] - .map(Coproduct[QueryValue[T]](_)) - .or(Decoder.decodeNonEmptyList[T].map(Coproduct[QueryValue[T]](_))) - implicit val decodeSingleValue: Decoder[SingleValue] = - Decoder.decodeDouble.map(Inl(_)).or(Decoder.decodeString.map(s ⇒ Inr(Inl(s)))) - - implicit val decodeState: Decoder[StateValue] = - Decoder.decodeString.emap { - case v if v == allW.value ⇒ Either.right(Coproduct[StateValue](allW.value)) - case v if v == activeW.value ⇒ Either.right(Coproduct[StateValue](activeW.value)) - case v if v == inactiveW.value ⇒ Either.right(Coproduct[StateValue](inactiveW.value)) - case _ ⇒ Either.left("unknown state defined") - } - - implicit val decodeQueryField: Decoder[QueryField] = new Decoder[QueryField] { - def apply(c: HCursor): Result[QueryField] = - c.downField("in").as[QueryValue[String]].right.map(_.fold(queryFieldF)) - } + Decoder.decodeJsonNumber.map(Inl(_)) or Decoder.decodeString.map(s ⇒ Inr(Inl(s))) - implicit val decodeCompoundValue: Decoder[CompoundValue] = decodeQueryValue[SingleValue] + implicit val decodeMultipleValue: Decoder[MultipleValues] = + Decoder.decodeNonEmptyList[JsonNumber].map(Inl(_)) or Decoder + .decodeNonEmptyList[String] + .map(s ⇒ Inr(Inl(s))) - implicit val decodeRange: Decoder[RangeValue] = new Decoder[RangeValue] { - def apply(c: HCursor): Result[RangeValue] = ??? // TODO: implement decoder - } + implicit val decodeCompoundValue: Decoder[CompoundValue] = + Decoder[SingleValue].map(Coproduct[CompoundValue](_)) or + Decoder[MultipleValues].map(Coproduct[CompoundValue](_)) - private[this] val decoderMap = Map( - "contains" → Decoder[Contains], - "matches" → Decoder[Matches], - "is" → Decoder[Is], - "range" → Decoder[Range], - "state" → Decoder[State] - ) - - implicit val decodeQueryFunction: Decoder[QueryFunction] = new Decoder[QueryFunction] { - def apply(c: HCursor): Result[QueryFunction] = - for { - tpe ← c.downField("type").as[String] - decoder ← decoderMap - .get(tpe) - .map(Either.right(_)) - .getOrElse(Either.left(DecodingFailure("", c.history))) - value ← decoder(c) - } yield value - } + implicit val decodeRange: Decoder[RangeValue] = + Decoder.decodeMapLike[Map, RangeFunction, JsonNumber].map(Inl(_)) or + Decoder.decodeMapLike[Map, RangeFunction, String].map(sm ⇒ Inr(Inl(sm))) - final case class Contains(in: QueryField, value: CompoundValue) extends QueryFunction { - type Value = CompoundValue - } - final case class Matches(in: QueryField, value: CompoundValue) extends QueryFunction { - type Value = CompoundValue - } - final case class Range(in: QueryField.Single, value: RangeValue) extends QueryFunction { - type Value = RangeValue - } - final case class Is(in: QueryField, value: CompoundValue) extends QueryFunction { - type Value = CompoundValue - } - final case class State(value: StateValue) extends QueryFunction { - type Value = StateValue - } + final case class contains(in: QueryField, value: CompoundValue) extends QueryFunction + final case class matches(in: QueryField, value: CompoundValue) extends QueryFunction + final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction + final case class is(in: QueryField, value: CompoundValue) extends QueryFunction + final case class state(value: EntityState) extends QueryFunction + + implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } final case class FCQuery(query: NonEmptyList[QueryFunction]) object FCQuery { - // TODO: check performance (e.g. no doubled attempt for query function decoding) if query is malformed - // should be fast, as it'd either fail fast on lack of array - // or on first (or single) malformed query function implicit val decodeFCQuery: Decoder[FCQuery] = Decoder .decodeNonEmptyList[QueryFunction] diff --git a/search-service/core/app/foxcomm/search/payload.scala b/search-service/core/app/foxcomm/search/payload.scala index 63f08437f8..3cad16237c 100644 --- a/search-service/core/app/foxcomm/search/payload.scala +++ b/search-service/core/app/foxcomm/search/payload.scala @@ -7,6 +7,6 @@ sealed trait SearchQuery { def fields: Option[NonEmptyList[String]] } object SearchQuery { - final case class ES(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchQuery - final case class FC(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchQuery + final case class es(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchQuery + final case class fc(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchQuery } From 3d3fe62653a936d78add593afe6b1a23966d651d Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 10:19:44 +0200 Subject: [PATCH 04/61] Add basic test --- search-service/build.sbt | 2 +- .../core/test/foxcomm/search/DslTest.scala | 46 +++++++++++++++++++ .../core/test/resources/happy_path.json | 33 +++++++++++++ search-service/project/Dependencies.scala | 7 +++ search-service/project/Settings.scala | 4 +- 5 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 search-service/core/test/foxcomm/search/DslTest.scala create mode 100644 search-service/core/test/resources/happy_path.json diff --git a/search-service/build.sbt b/search-service/build.sbt index b535ff40b8..8690e41754 100644 --- a/search-service/build.sbt +++ b/search-service/build.sbt @@ -8,7 +8,7 @@ version := "0.1-SNAPSHOT" lazy val core = (project in file("core")) .settings(Settings.common) .settings( - libraryDependencies ++= Dependencies.core ++ Dependencies.es ++ Dependencies.circe + libraryDependencies ++= Dependencies.core ++ Dependencies.es ++ Dependencies.circe ++ Dependencies.test.core ) lazy val finch = (project in file("finch")) diff --git a/search-service/core/test/foxcomm/search/DslTest.scala b/search-service/core/test/foxcomm/search/DslTest.scala new file mode 100644 index 0000000000..dfb6255eee --- /dev/null +++ b/search-service/core/test/foxcomm/search/DslTest.scala @@ -0,0 +1,46 @@ +package foxcomm.search + +import io.circe.JsonNumber +import io.circe.generic.extras.auto._ +import io.circe.parser._ +import org.scalatest.EitherValues._ +import org.scalatest.OptionValues._ +import org.scalatest.{Assertion, FlatSpec, Matchers} +import scala.io.Source +import shapeless._ +import shapeless.syntax.typeable._ + +class DslTest extends FlatSpec with Matchers { + def assertQueryFunction[T <: QueryFunction: Typeable](qf: QueryFunction)( + assertion: T ⇒ Assertion): Assertion = + assertion(qf.cast[T].value) + + "DSL" should "parse multiple queries" in { + val json = + parse(Source.fromInputStream(getClass.getResourceAsStream("/happy_path.json")).mkString).right.value + val queries = json.as[SearchQuery.fc].right.value.query.query.toList + assertQueryFunction[QueryFunction.is](queries.head) { is ⇒ + is.in.toList should === (List("slug")) + is.value.fold(QueryFunction.listOfAnyValueF) should === (List("awesome", "whatever")) + } + assertQueryFunction[QueryFunction.state](queries(1)) { state ⇒ + state.value should === (EntityState.all) + } + assertQueryFunction[QueryFunction.contains](queries(2)) { contains ⇒ + contains.in.toList should === (List("tags")) + contains.value.fold(QueryFunction.listOfAnyValueF) should === (List("gift")) + } + assertQueryFunction[QueryFunction.matches](queries(3)) { matches ⇒ + matches.in.toList should === (List("title", "description")) + matches.value.fold(QueryFunction.listOfAnyValueF) should === (List("food", "drink")) + } + assertQueryFunction[QueryFunction.range](queries(4)) { range ⇒ + range.in.toList should === (List("price")) + range.value.unify.cast[Map[RangeFunction, JsonNumber]].value.mapValues(_.toString) should === ( + Map( + RangeFunction.Lt → "5000", + RangeFunction.Gte → "1000" + )) + } + } +} diff --git a/search-service/core/test/resources/happy_path.json b/search-service/core/test/resources/happy_path.json new file mode 100644 index 0000000000..8ced429438 --- /dev/null +++ b/search-service/core/test/resources/happy_path.json @@ -0,0 +1,33 @@ +{ + "type": "fc", + "fields": [ "id" ], + "query": [ + { + "type": "is", + "in": "slug", + "value": [ "awesome", "whatever" ] + }, + { + "type": "state", + "value": "all" + }, + { + "type": "contains", + "in": [ "tags" ], + "value": "gift" + }, + { + "type": "matches", + "in": [ "title", "description" ], + "value": [ "food", "drink" ] + }, + { + "type": "range", + "in": "price", + "value": { + "<": 5000, + "gte": 1000 + } + } + ] +} diff --git a/search-service/project/Dependencies.scala b/search-service/project/Dependencies.scala index 7d32fd03c7..8600abe3c7 100644 --- a/search-service/project/Dependencies.scala +++ b/search-service/project/Dependencies.scala @@ -32,4 +32,11 @@ object Dependencies { ) val jwt = "com.pauldijou" %% "jwt-core" % "0.12.1" + + object test { + def core = + Seq( + "org.scalatest" %% "scalatest" % "3.0.3" + ).map(_ % "test") + } } diff --git a/search-service/project/Settings.scala b/search-service/project/Settings.scala index c2a2d56cb8..4609602ac4 100644 --- a/search-service/project/Settings.scala +++ b/search-service/project/Settings.scala @@ -29,7 +29,9 @@ object Settings { Wart.Nothing, Wart.PublicInference), scalaSource in Compile := baseDirectory.value / "app", - resourceDirectory in Compile := baseDirectory.value / "resources" + resourceDirectory in Compile := baseDirectory.value / "resources", + scalaSource in Test := baseDirectory.value / "test", + resourceDirectory in Test := baseDirectory.value / "test" / "resources" ) def deploy: Seq[Def.Setting[_]] = Seq( From a39a8b68015f5c36624b5cbfad8f359f034fa1aa Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 10:24:58 +0200 Subject: [PATCH 05/61] Add search-service to builder --- builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/builder.py b/builder.py index b149c41fed..82719e2dab 100755 --- a/builder.py +++ b/builder.py @@ -49,6 +49,7 @@ 'onboarding/ui', 'phoenix-scala', 'phoenix-scala/seeder', + 'search-service', 'solomon', 'tabernacle/docker/neo4j', 'tabernacle/docker/neo4j_reset', From 6466188cc883a1efd4fe64138be0c7248f7ae6e0 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 10:32:10 +0200 Subject: [PATCH 06/61] Adjust Makefile --- search-service/Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/search-service/Makefile b/search-service/Makefile index e57e5f2707..e1117bff4b 100644 --- a/search-service/Makefile +++ b/search-service/Makefile @@ -8,7 +8,7 @@ SBT_CMD = sbt -DDOCKER_REPO=$(DOCKER_REPO) -DDOCKER_TAG=${DOCKER_TAG} clean: $(call header, Cleaning) - ${SBT_CMD} 'clean' + ${SBT_CMD} '; clean' build: $(call header, Building) @@ -16,15 +16,15 @@ build: test: $(call header, Testing) - ${SBT_CMD} 'test' + ${SBT_CMD} '; test' docker: $(call header, Dockerizing) - ${SBT_CMD} 'api/docker' + ${SBT_CMD} '; api/docker' docker-push: $(call header, Registering) - ${SBT_CMD} 'api/dockerPush' + ${SBT_CMD} '; api/dockerPush' -docker-build: - ${SBT_CMD} 'api/dockerBuildAndPush' +docker-all: + ${SBT_CMD} '; api/dockerBuildAndPush' From 1fff1791e7bda1d5aad3a379f927849a75341f92 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 11:29:30 +0200 Subject: [PATCH 07/61] Add neq function --- search-service/core/app/foxcomm/search/SearchService.scala | 4 +++- search-service/core/app/foxcomm/search/dsl.scala | 3 ++- search-service/core/test/foxcomm/search/DslTest.scala | 2 +- search-service/core/test/resources/happy_path.json | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index 0fa6a37cbd..7c08ed1779 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -21,8 +21,10 @@ class SearchService(private val client: ElasticClient) extends AnyVal { case SearchQuery.fc(query, _) ⇒ (_: SearchDefinition) bool { query.query.foldLeft(new BoolQueryDefinition) { - case (bool, QueryFunction.is(in, value)) ⇒ + case (bool, QueryFunction.eq(in, value)) ⇒ bool.filter(in.toList.map(termsQuery(_, value.fold(QueryFunction.listOfAnyValueF): _*))) + case (bool, QueryFunction.neq(in, value)) ⇒ + bool.not(in.toList.map(termsQuery(_, value.fold(QueryFunction.listOfAnyValueF): _*))) case (bool, _) ⇒ bool // TODO: implement rest of cases } } diff --git a/search-service/core/app/foxcomm/search/dsl.scala b/search-service/core/app/foxcomm/search/dsl.scala index 48689e63bf..7b27606b1b 100644 --- a/search-service/core/app/foxcomm/search/dsl.scala +++ b/search-service/core/app/foxcomm/search/dsl.scala @@ -91,7 +91,8 @@ object QueryFunction { final case class contains(in: QueryField, value: CompoundValue) extends QueryFunction final case class matches(in: QueryField, value: CompoundValue) extends QueryFunction final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction - final case class is(in: QueryField, value: CompoundValue) extends QueryFunction + final case class eq(in: QueryField, value: CompoundValue) extends QueryFunction + final case class neq(in: QueryField, value: CompoundValue) extends QueryFunction final case class state(value: EntityState) extends QueryFunction implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] diff --git a/search-service/core/test/foxcomm/search/DslTest.scala b/search-service/core/test/foxcomm/search/DslTest.scala index dfb6255eee..7ccf681e68 100644 --- a/search-service/core/test/foxcomm/search/DslTest.scala +++ b/search-service/core/test/foxcomm/search/DslTest.scala @@ -19,7 +19,7 @@ class DslTest extends FlatSpec with Matchers { val json = parse(Source.fromInputStream(getClass.getResourceAsStream("/happy_path.json")).mkString).right.value val queries = json.as[SearchQuery.fc].right.value.query.query.toList - assertQueryFunction[QueryFunction.is](queries.head) { is ⇒ + assertQueryFunction[QueryFunction.eq](queries.head) { is ⇒ is.in.toList should === (List("slug")) is.value.fold(QueryFunction.listOfAnyValueF) should === (List("awesome", "whatever")) } diff --git a/search-service/core/test/resources/happy_path.json b/search-service/core/test/resources/happy_path.json index 8ced429438..01e1718260 100644 --- a/search-service/core/test/resources/happy_path.json +++ b/search-service/core/test/resources/happy_path.json @@ -3,7 +3,7 @@ "fields": [ "id" ], "query": [ { - "type": "is", + "type": "eq", "in": "slug", "value": [ "awesome", "whatever" ] }, From 6eef884f222aa122062e717f14467ba8aae970f8 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 16:16:37 +0200 Subject: [PATCH 08/61] Move query dsl --- .../api/app/foxcomm/search/api/Api.scala | 4 +- .../app/foxcomm/search/SearchService.scala | 7 +- .../core/app/foxcomm/search/dsl.scala | 108 ---------------- .../core/app/foxcomm/search/dsl/package.scala | 7 ++ .../core/app/foxcomm/search/dsl/query.scala | 112 +++++++++++++++++ .../core/app/foxcomm/search/payload.scala | 9 +- .../{DslTest.scala => dsl/QueryDslSpec.scala} | 8 +- search-service/core/test/resources/es.json | 116 ++++++++++++++++++ search-service/core/test/resources/fc.json | 0 .../core/test/resources/happy_path.json | 60 +++++---- 10 files changed, 278 insertions(+), 153 deletions(-) delete mode 100644 search-service/core/app/foxcomm/search/dsl.scala create mode 100644 search-service/core/app/foxcomm/search/dsl/package.scala create mode 100644 search-service/core/app/foxcomm/search/dsl/query.scala rename search-service/core/test/foxcomm/search/{DslTest.scala => dsl/QueryDslSpec.scala} (90%) create mode 100644 search-service/core/test/resources/es.json create mode 100644 search-service/core/test/resources/fc.json diff --git a/search-service/api/app/foxcomm/search/api/Api.scala b/search-service/api/app/foxcomm/search/api/Api.scala index 097d5969fb..ebfa2d8393 100644 --- a/search-service/api/app/foxcomm/search/api/Api.scala +++ b/search-service/api/app/foxcomm/search/api/Api.scala @@ -16,8 +16,8 @@ object Api extends App { def endpoint(searchService: SearchService)(implicit ec: ExecutionContext) = post( "search" :: string :: string :: param("size") - .as[Int] :: paramOption("from").as[Int] :: jsonBody[SearchQuery]) { - (searchIndex: String, searchType: String, size: Int, from: Option[Int], searchQuery: SearchQuery) ⇒ + .as[Int] :: paramOption("from").as[Int] :: jsonBody[SearchPayload]) { + (searchIndex: String, searchType: String, size: Int, from: Option[Int], searchQuery: SearchPayload) ⇒ searchService .searchFor(searchIndex / searchType, searchQuery, searchSize = size, searchFrom = from) .toTwitterFuture diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index 7c08ed1779..5218ab4907 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -4,6 +4,7 @@ import scala.language.postfixOps import cats.implicits._ import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s._ +import foxcomm.search.dsl.query.QueryFunction import io.circe._ import io.circe.jawn.parseByteBuffer import org.elasticsearch.common.settings.Settings @@ -13,12 +14,12 @@ class SearchService(private val client: ElasticClient) extends AnyVal { import SearchService.ExtractJsonObject def searchFor(searchIndex: IndexAndTypes, - searchQuery: SearchQuery, + searchQuery: SearchPayload, searchSize: Int, searchFrom: Option[Int])(implicit ec: ExecutionContext): Future[SearchResult] = { val withQuery = searchQuery match { - case SearchQuery.es(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces - case SearchQuery.fc(query, _) ⇒ + case SearchPayload.es(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces + case SearchPayload.fc(query, _) ⇒ (_: SearchDefinition) bool { query.query.foldLeft(new BoolQueryDefinition) { case (bool, QueryFunction.eq(in, value)) ⇒ diff --git a/search-service/core/app/foxcomm/search/dsl.scala b/search-service/core/app/foxcomm/search/dsl.scala deleted file mode 100644 index 7b27606b1b..0000000000 --- a/search-service/core/app/foxcomm/search/dsl.scala +++ /dev/null @@ -1,108 +0,0 @@ -package foxcomm.search - -import cats.data.NonEmptyList -import cats.syntax.either._ -import io.circe.generic.extras.semiauto._ -import io.circe.{Decoder, JsonNumber, KeyDecoder} -import shapeless._ - -sealed trait QueryField { - def toList: List[String] -} -object QueryField { - final case class Single(field: String) extends QueryField { - def toList: List[String] = List(field) - } - object Single { - implicit val decodeSingle: Decoder[Single] = Decoder.decodeString - .emap { - case s if s.startsWith("$") ⇒ Either.left(s"Defined unknown special query field $s") - case s ⇒ Either.right(s) - } - .map(Single(_)) - } - - final case class Multiple(fields: NonEmptyList[String]) extends QueryField { - def toList: List[String] = fields.toList - } - object Multiple { - implicit val decodeMultiple: Decoder[Multiple] = - Decoder.decodeNonEmptyList[String].map(Multiple(_)) - } - - implicit val decodeQueryField: Decoder[QueryField] = - Decoder[Single].map(s ⇒ s: QueryField) or - Decoder[Multiple].map(m ⇒ m: QueryField) -} - -sealed trait RangeFunction -object RangeFunction { - case object Lt extends RangeFunction - case object Lte extends RangeFunction - case object Gt extends RangeFunction - case object Gte extends RangeFunction - - implicit val decodeRangeFunction: KeyDecoder[RangeFunction] = KeyDecoder.instance { - case "lt" | "<" ⇒ Some(Lt) - case "lte" | "<=" ⇒ Some(Lte) - case "gt" | ">" ⇒ Some(Gt) - case "gte" | ">=" ⇒ Some(Gte) - } -} - -sealed trait EntityState -object EntityState { - case object all extends EntityState - case object active extends EntityState - case object inactive extends EntityState - - implicit val decodeEntityState: Decoder[EntityState] = deriveEnumerationDecoder[EntityState] -} - -sealed trait QueryFunction -@SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.ExplicitImplicitTypes")) -object QueryFunction { - type SingleValue = JsonNumber :+: String :+: CNil - type MultipleValues = NonEmptyList[JsonNumber] :+: NonEmptyList[String] :+: CNil - type CompoundValue = SingleValue :+: MultipleValues :+: CNil - type RangeValue = Map[RangeFunction, JsonNumber] :+: Map[RangeFunction, String] :+: CNil - - object listOfAnyValueF extends Poly1 { - implicit def caseValue = at[SingleValue](v ⇒ List(v.unify.asInstanceOf[AnyRef])) - implicit def caseValues = at[MultipleValues](_.unify.toList.asInstanceOf[List[AnyRef]]) - } - - implicit val decodeSingleValue: Decoder[SingleValue] = - Decoder.decodeJsonNumber.map(Inl(_)) or Decoder.decodeString.map(s ⇒ Inr(Inl(s))) - - implicit val decodeMultipleValue: Decoder[MultipleValues] = - Decoder.decodeNonEmptyList[JsonNumber].map(Inl(_)) or Decoder - .decodeNonEmptyList[String] - .map(s ⇒ Inr(Inl(s))) - - implicit val decodeCompoundValue: Decoder[CompoundValue] = - Decoder[SingleValue].map(Coproduct[CompoundValue](_)) or - Decoder[MultipleValues].map(Coproduct[CompoundValue](_)) - - implicit val decodeRange: Decoder[RangeValue] = - Decoder.decodeMapLike[Map, RangeFunction, JsonNumber].map(Inl(_)) or - Decoder.decodeMapLike[Map, RangeFunction, String].map(sm ⇒ Inr(Inl(sm))) - - final case class contains(in: QueryField, value: CompoundValue) extends QueryFunction - final case class matches(in: QueryField, value: CompoundValue) extends QueryFunction - final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction - final case class eq(in: QueryField, value: CompoundValue) extends QueryFunction - final case class neq(in: QueryField, value: CompoundValue) extends QueryFunction - final case class state(value: EntityState) extends QueryFunction - - implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] -} - -final case class FCQuery(query: NonEmptyList[QueryFunction]) -object FCQuery { - implicit val decodeFCQuery: Decoder[FCQuery] = - Decoder - .decodeNonEmptyList[QueryFunction] - .or(Decoder[QueryFunction].map(NonEmptyList.of(_))) - .map(FCQuery(_)) -} diff --git a/search-service/core/app/foxcomm/search/dsl/package.scala b/search-service/core/app/foxcomm/search/dsl/package.scala new file mode 100644 index 0000000000..4f382b5917 --- /dev/null +++ b/search-service/core/app/foxcomm/search/dsl/package.scala @@ -0,0 +1,7 @@ +package foxcomm.search + +import io.circe.generic.extras.Configuration + +package object dsl { + implicit def configuration: Configuration = foxcomm.search.configuration +} diff --git a/search-service/core/app/foxcomm/search/dsl/query.scala b/search-service/core/app/foxcomm/search/dsl/query.scala new file mode 100644 index 0000000000..b50d630190 --- /dev/null +++ b/search-service/core/app/foxcomm/search/dsl/query.scala @@ -0,0 +1,112 @@ +package foxcomm.search.dsl + +import cats.data.NonEmptyList +import cats.syntax.either._ +import io.circe.generic.extras.semiauto._ +import io.circe.{Decoder, JsonNumber, KeyDecoder} +import shapeless._ + +object query { + sealed trait QueryField { + def toList: List[String] + } + object QueryField { + final case class Single(field: String) extends QueryField { + def toList: List[String] = List(field) + } + object Single { + implicit val decodeSingle: Decoder[Single] = Decoder.decodeString + .emap { + case s if s.startsWith("$") ⇒ Either.left(s"Defined unknown special query field $s") + case s ⇒ Either.right(s) + } + .map(Single(_)) + } + + final case class Multiple(fields: NonEmptyList[String]) extends QueryField { + def toList: List[String] = fields.toList + } + object Multiple { + implicit val decodeMultiple: Decoder[Multiple] = + Decoder.decodeNonEmptyList[String].map(Multiple(_)) + } + + implicit val decodeQueryField: Decoder[QueryField] = + Decoder[Single].map(s ⇒ s: QueryField) or + Decoder[Multiple].map(m ⇒ m: QueryField) + } + + sealed trait RangeFunction + object RangeFunction { + case object Lt extends RangeFunction + case object Lte extends RangeFunction + case object Gt extends RangeFunction + case object Gte extends RangeFunction + + implicit val decodeRangeFunction: KeyDecoder[RangeFunction] = KeyDecoder.instance { + case "lt" | "<" ⇒ Some(Lt) + case "lte" | "<=" ⇒ Some(Lte) + case "gt" | ">" ⇒ Some(Gt) + case "gte" | ">=" ⇒ Some(Gte) + } + } + + sealed trait EntityState + object EntityState { + case object all extends EntityState + case object active extends EntityState + case object inactive extends EntityState + + implicit val decodeEntityState: Decoder[EntityState] = deriveEnumerationDecoder[EntityState] + } + + sealed trait QueryFunction + @SuppressWarnings( + Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.ExplicitImplicitTypes")) + object QueryFunction { + type SingleValue = JsonNumber :+: String :+: CNil + type MultipleValues = NonEmptyList[JsonNumber] :+: NonEmptyList[String] :+: CNil + type CompoundValue = SingleValue :+: MultipleValues :+: CNil + type RangeValue = Map[RangeFunction, JsonNumber] :+: Map[RangeFunction, String] :+: CNil + + object listOfAnyValueF extends Poly1 { + implicit def caseValue = at[SingleValue](v ⇒ List(v.unify.asInstanceOf[AnyRef])) + + implicit def caseValues = at[MultipleValues](_.unify.toList.asInstanceOf[List[AnyRef]]) + } + + implicit val decodeSingleValue: Decoder[SingleValue] = + Decoder.decodeJsonNumber.map(Inl(_)) or Decoder.decodeString.map(s ⇒ Inr(Inl(s))) + + implicit val decodeMultipleValue: Decoder[MultipleValues] = + Decoder.decodeNonEmptyList[JsonNumber].map(Inl(_)) or Decoder + .decodeNonEmptyList[String] + .map(s ⇒ Inr(Inl(s))) + + implicit val decodeCompoundValue: Decoder[CompoundValue] = + Decoder[SingleValue].map(Coproduct[CompoundValue](_)) or + Decoder[MultipleValues].map(Coproduct[CompoundValue](_)) + + implicit val decodeRange: Decoder[RangeValue] = + Decoder.decodeMapLike[Map, RangeFunction, JsonNumber].map(Inl(_)) or + Decoder.decodeMapLike[Map, RangeFunction, String].map(sm ⇒ Inr(Inl(sm))) + + final case class contains(in: QueryField, value: CompoundValue) extends QueryFunction + final case class matches(in: QueryField, value: CompoundValue) extends QueryFunction + final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction + final case class eq(in: QueryField, value: CompoundValue) extends QueryFunction + final case class neq(in: QueryField, value: CompoundValue) extends QueryFunction + final case class state(value: EntityState) extends QueryFunction + + implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] + } + + final case class FCQuery(query: NonEmptyList[QueryFunction]) + object FCQuery { + implicit val decodeFCQuery: Decoder[FCQuery] = + Decoder + .decodeNonEmptyList[QueryFunction] + .or(Decoder[QueryFunction].map(NonEmptyList.of(_))) + .map(FCQuery(_)) + } +} diff --git a/search-service/core/app/foxcomm/search/payload.scala b/search-service/core/app/foxcomm/search/payload.scala index 3cad16237c..36367c7bc5 100644 --- a/search-service/core/app/foxcomm/search/payload.scala +++ b/search-service/core/app/foxcomm/search/payload.scala @@ -1,12 +1,13 @@ package foxcomm.search import cats.data.NonEmptyList +import foxcomm.search.dsl.query._ import io.circe.JsonObject -sealed trait SearchQuery { +sealed trait SearchPayload { def fields: Option[NonEmptyList[String]] } -object SearchQuery { - final case class es(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchQuery - final case class fc(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchQuery +object SearchPayload { + final case class es(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchPayload + final case class fc(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchPayload } diff --git a/search-service/core/test/foxcomm/search/DslTest.scala b/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala similarity index 90% rename from search-service/core/test/foxcomm/search/DslTest.scala rename to search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala index 7ccf681e68..8f3569a068 100644 --- a/search-service/core/test/foxcomm/search/DslTest.scala +++ b/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala @@ -1,7 +1,7 @@ -package foxcomm.search +package foxcomm.search.dsl +import foxcomm.search.dsl.query._ import io.circe.JsonNumber -import io.circe.generic.extras.auto._ import io.circe.parser._ import org.scalatest.EitherValues._ import org.scalatest.OptionValues._ @@ -10,7 +10,7 @@ import scala.io.Source import shapeless._ import shapeless.syntax.typeable._ -class DslTest extends FlatSpec with Matchers { +class QueryDslSpec extends FlatSpec with Matchers { def assertQueryFunction[T <: QueryFunction: Typeable](qf: QueryFunction)( assertion: T ⇒ Assertion): Assertion = assertion(qf.cast[T].value) @@ -18,7 +18,7 @@ class DslTest extends FlatSpec with Matchers { "DSL" should "parse multiple queries" in { val json = parse(Source.fromInputStream(getClass.getResourceAsStream("/happy_path.json")).mkString).right.value - val queries = json.as[SearchQuery.fc].right.value.query.query.toList + val queries = json.as[FCQuery].right.value.query.toList assertQueryFunction[QueryFunction.eq](queries.head) { is ⇒ is.in.toList should === (List("slug")) is.value.fold(QueryFunction.listOfAnyValueF) should === (List("awesome", "whatever")) diff --git a/search-service/core/test/resources/es.json b/search-service/core/test/resources/es.json new file mode 100644 index 0000000000..4d54808fb0 --- /dev/null +++ b/search-service/core/test/resources/es.json @@ -0,0 +1,116 @@ +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "context": "default" + } + } + ], + "must": [ + { + "nested": { + "path": "taxonomies", + "query": { + "bool": { + "must": [ + { + "query": { + "bool": { + "should": { + "term": { + "taxonomies.taxons": "MEN" + } + } + } + } + } + ] + } + } + } + }, + { + "nested": { + "path": "taxonomies", + "query": { + "bool": { + "must": [ + { + "query": { + "bool": { + "should": { + "term": { + "taxonomies.taxons": "SHOES" + } + } + } + } + } + ] + } + } + } + }, + { + "nested": { + "path": "taxonomies", + "query": { + "bool": { + "must": [ + { + "query": { + "bool": { + "should": { + "term": { + "taxonomies.taxons": "OUTDOOR" + } + } + } + } + } + ] + } + } + } + } + ], + "must_not": [ + { + "term": { + "tags": "GIFT-CARD" + } + } + ] + } + }, + "sort": [ + { + "salePrice": { + "order": "asc" + } + } + ], + "aggs": { + "taxonomies": { + "nested": { + "path": "taxonomies" + }, + "aggs": { + "taxonomy": { + "terms": { + "field": "taxonomies.taxonomy" + }, + "aggs": { + "taxon": { + "terms": { + "field": "taxonomies.taxons" + } + } + } + } + } + } + } +} diff --git a/search-service/core/test/resources/fc.json b/search-service/core/test/resources/fc.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/search-service/core/test/resources/happy_path.json b/search-service/core/test/resources/happy_path.json index 01e1718260..63ad5861dd 100644 --- a/search-service/core/test/resources/happy_path.json +++ b/search-service/core/test/resources/happy_path.json @@ -1,33 +1,29 @@ -{ - "type": "fc", - "fields": [ "id" ], - "query": [ - { - "type": "eq", - "in": "slug", - "value": [ "awesome", "whatever" ] - }, - { - "type": "state", - "value": "all" - }, - { - "type": "contains", - "in": [ "tags" ], - "value": "gift" - }, - { - "type": "matches", - "in": [ "title", "description" ], - "value": [ "food", "drink" ] - }, - { - "type": "range", - "in": "price", - "value": { - "<": 5000, - "gte": 1000 - } +[ + { + "type": "eq", + "in": "slug", + "value": [ "awesome", "whatever" ] + }, + { + "type": "state", + "value": "all" + }, + { + "type": "contains", + "in": [ "tags" ], + "value": "gift" + }, + { + "type": "matches", + "in": [ "title", "description" ], + "value": [ "food", "drink" ] + }, + { + "type": "range", + "in": "price", + "value": { + "<": 5000, + "gte": 1000 } - ] -} + } +] From ca6c6277dd8d58d5db3b28d10c4b628f33a7bdc2 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 18:34:58 +0200 Subject: [PATCH 09/61] Interpret match function Add simple match function implementation. Add temp ugly workaround for issue with shapeless Folder. --- .../app/foxcomm/search/SearchService.scala | 9 +- .../core/app/foxcomm/search/dsl/query.scala | 103 ++++++++++++------ .../foxcomm/search/dsl/QueryDslSpec.scala | 12 +- .../core/test/resources/happy_path.json | 5 - 4 files changed, 80 insertions(+), 49 deletions(-) diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index 5218ab4907..85e37c2291 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -4,7 +4,7 @@ import scala.language.postfixOps import cats.implicits._ import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s._ -import foxcomm.search.dsl.query.QueryFunction +import foxcomm.search.dsl.query._ import io.circe._ import io.circe.jawn.parseByteBuffer import org.elasticsearch.common.settings.Settings @@ -23,9 +23,12 @@ class SearchService(private val client: ElasticClient) extends AnyVal { (_: SearchDefinition) bool { query.query.foldLeft(new BoolQueryDefinition) { case (bool, QueryFunction.eq(in, value)) ⇒ - bool.filter(in.toList.map(termsQuery(_, value.fold(QueryFunction.listOfAnyValueF): _*))) + bool.filter(in.toList.map(termsQuery(_, value.toList: _*))) case (bool, QueryFunction.neq(in, value)) ⇒ - bool.not(in.toList.map(termsQuery(_, value.fold(QueryFunction.listOfAnyValueF): _*))) + bool.not(in.toList.map(termsQuery(_, value.toList: _*))) + case (bool, QueryFunction.matches(in, value)) ⇒ + val fields = in.toList + bool.must(value.toList.map(q ⇒ multiMatchQuery(q).fields(fields))) case (bool, _) ⇒ bool // TODO: implement rest of cases } } diff --git a/search-service/core/app/foxcomm/search/dsl/query.scala b/search-service/core/app/foxcomm/search/dsl/query.scala index b50d630190..164becb21e 100644 --- a/search-service/core/app/foxcomm/search/dsl/query.scala +++ b/search-service/core/app/foxcomm/search/dsl/query.scala @@ -5,7 +5,9 @@ import cats.syntax.either._ import io.circe.generic.extras.semiauto._ import io.circe.{Decoder, JsonNumber, KeyDecoder} import shapeless._ +import shapeless.ops.coproduct.Folder +@SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.ExplicitImplicitTypes")) object query { sealed trait QueryField { def toList: List[String] @@ -38,10 +40,21 @@ object query { sealed trait RangeFunction object RangeFunction { - case object Lt extends RangeFunction - case object Lte extends RangeFunction - case object Gt extends RangeFunction - case object Gte extends RangeFunction + sealed trait LowerBound extends RangeFunction { + override def hashCode(): Int = super.hashCode() + + override def equals(obj: scala.Any): Boolean = super.equals(obj) + } + case object Lt extends RangeFunction with LowerBound + case object Lte extends RangeFunction with LowerBound + + sealed trait UpperBound extends RangeFunction { + override def hashCode(): Int = super.hashCode() + + override def equals(obj: scala.Any): Boolean = super.equals(obj) + } + case object Gt extends RangeFunction with UpperBound + case object Gte extends RangeFunction with UpperBound implicit val decodeRangeFunction: KeyDecoder[RangeFunction] = KeyDecoder.instance { case "lt" | "<" ⇒ Some(Lt) @@ -60,43 +73,67 @@ object query { implicit val decodeEntityState: Decoder[EntityState] = deriveEnumerationDecoder[EntityState] } - sealed trait QueryFunction - @SuppressWarnings( - Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.ExplicitImplicitTypes")) - object QueryFunction { - type SingleValue = JsonNumber :+: String :+: CNil - type MultipleValues = NonEmptyList[JsonNumber] :+: NonEmptyList[String] :+: CNil - type CompoundValue = SingleValue :+: MultipleValues :+: CNil - type RangeValue = Map[RangeFunction, JsonNumber] :+: Map[RangeFunction, String] :+: CNil + type QueryValue[T] = T :+: NonEmptyList[T] :+: CNil + type CompoundValue = QueryValue[JsonNumber] :+: QueryValue[String] :+: CNil + type RangeValue = Map[RangeFunction, JsonNumber] :+: Map[RangeFunction, String] :+: CNil + + object queryValueF extends Poly1 { + implicit def singleValue[T] = at[T](List(_)) + + implicit def multipleValues[T] = at[NonEmptyList[T]](_.toList) + } - object listOfAnyValueF extends Poly1 { - implicit def caseValue = at[SingleValue](v ⇒ List(v.unify.asInstanceOf[AnyRef])) + object listOfAnyValueF extends Poly1 { + implicit def caseJsonNumber = at[QueryValue[JsonNumber]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) - implicit def caseValues = at[MultipleValues](_.unify.toList.asInstanceOf[List[AnyRef]]) + implicit def caseString = at[QueryValue[String]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) + } + + // Ugly optimisation to avoid recreation of folder instance on each implicit call + // It is safe to cast it the implicit to proper type, + // since we don't assume anything about the type, but only construct list. + private[this] val queryValueFolderInstance = new Folder[queryValueF.type, QueryValue[_]] { + type Out = List[_] + + def apply(qv: QueryValue[_]): Out = qv match { + case Inl(v) ⇒ List(v) + case Inr(Inl(vs)) ⇒ vs.toList + case _ ⇒ Nil } + } + + // TODO: for some reason shapeless cannot find implicit `Folder` instance + // if Poly function cases contain generic param as in `queryValueF` + implicit def queryValueFolder[T]: Folder[queryValueF.type, QueryValue[T]] = + queryValueFolderInstance.asInstanceOf[Folder[queryValueF.type, QueryValue[T]]] - implicit val decodeSingleValue: Decoder[SingleValue] = - Decoder.decodeJsonNumber.map(Inl(_)) or Decoder.decodeString.map(s ⇒ Inr(Inl(s))) + implicit class RichQueryValue[T](val qv: QueryValue[T]) extends AnyVal { + def toList: List[T] = qv.fold(queryValueF).asInstanceOf[List[T]] + } + + implicit class RichCompoundValue(val cv: CompoundValue) extends AnyVal { + def toList(implicit folder: Folder[listOfAnyValueF.type, CompoundValue]): List[AnyRef] = + cv.fold(listOfAnyValueF).asInstanceOf[List[AnyRef]] + } - implicit val decodeMultipleValue: Decoder[MultipleValues] = - Decoder.decodeNonEmptyList[JsonNumber].map(Inl(_)) or Decoder - .decodeNonEmptyList[String] - .map(s ⇒ Inr(Inl(s))) + implicit def decodeQueryValue[T: Decoder]: Decoder[QueryValue[T]] = + Decoder[T].map(Inl(_)) or Decoder.decodeNonEmptyList[T].map(n ⇒ Inr(Inl(n))) - implicit val decodeCompoundValue: Decoder[CompoundValue] = - Decoder[SingleValue].map(Coproduct[CompoundValue](_)) or - Decoder[MultipleValues].map(Coproduct[CompoundValue](_)) + implicit val decodeCompoundValue: Decoder[CompoundValue] = + Decoder[QueryValue[JsonNumber]].map(Coproduct[CompoundValue](_)) or + Decoder[QueryValue[String]].map(Coproduct[CompoundValue](_)) - implicit val decodeRange: Decoder[RangeValue] = - Decoder.decodeMapLike[Map, RangeFunction, JsonNumber].map(Inl(_)) or - Decoder.decodeMapLike[Map, RangeFunction, String].map(sm ⇒ Inr(Inl(sm))) + implicit val decodeRange: Decoder[RangeValue] = + Decoder.decodeMapLike[Map, RangeFunction, JsonNumber].map(Inl(_)) or + Decoder.decodeMapLike[Map, RangeFunction, String].map(sm ⇒ Inr(Inl(sm))) - final case class contains(in: QueryField, value: CompoundValue) extends QueryFunction - final case class matches(in: QueryField, value: CompoundValue) extends QueryFunction - final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction - final case class eq(in: QueryField, value: CompoundValue) extends QueryFunction - final case class neq(in: QueryField, value: CompoundValue) extends QueryFunction - final case class state(value: EntityState) extends QueryFunction + sealed trait QueryFunction + object QueryFunction { + final case class matches(in: QueryField, value: QueryValue[String]) extends QueryFunction + final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction + final case class eq(in: QueryField, value: CompoundValue) extends QueryFunction + final case class neq(in: QueryField, value: CompoundValue) extends QueryFunction + final case class state(value: EntityState) extends QueryFunction implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } diff --git a/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala b/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala index 8f3569a068..7a86dcb75d 100644 --- a/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala +++ b/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala @@ -21,20 +21,16 @@ class QueryDslSpec extends FlatSpec with Matchers { val queries = json.as[FCQuery].right.value.query.toList assertQueryFunction[QueryFunction.eq](queries.head) { is ⇒ is.in.toList should === (List("slug")) - is.value.fold(QueryFunction.listOfAnyValueF) should === (List("awesome", "whatever")) + is.value.toList should === (List("awesome", "whatever")) } assertQueryFunction[QueryFunction.state](queries(1)) { state ⇒ state.value should === (EntityState.all) } - assertQueryFunction[QueryFunction.contains](queries(2)) { contains ⇒ - contains.in.toList should === (List("tags")) - contains.value.fold(QueryFunction.listOfAnyValueF) should === (List("gift")) - } - assertQueryFunction[QueryFunction.matches](queries(3)) { matches ⇒ + assertQueryFunction[QueryFunction.matches](queries(2)) { matches ⇒ matches.in.toList should === (List("title", "description")) - matches.value.fold(QueryFunction.listOfAnyValueF) should === (List("food", "drink")) + matches.value.toList should === (List("food", "drink")) } - assertQueryFunction[QueryFunction.range](queries(4)) { range ⇒ + assertQueryFunction[QueryFunction.range](queries(3)) { range ⇒ range.in.toList should === (List("price")) range.value.unify.cast[Map[RangeFunction, JsonNumber]].value.mapValues(_.toString) should === ( Map( diff --git a/search-service/core/test/resources/happy_path.json b/search-service/core/test/resources/happy_path.json index 63ad5861dd..0a425ed9a8 100644 --- a/search-service/core/test/resources/happy_path.json +++ b/search-service/core/test/resources/happy_path.json @@ -8,11 +8,6 @@ "type": "state", "value": "all" }, - { - "type": "contains", - "in": [ "tags" ], - "value": "gift" - }, { "type": "matches", "in": [ "title", "description" ], From d427ed1b79e42b5cd20bad88a841c7a293b63df3 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 13 Jun 2017 19:21:40 +0200 Subject: [PATCH 10/61] Add simple range function implementation --- .../app/foxcomm/search/SearchService.scala | 13 +++++ .../core/app/foxcomm/search/dsl/query.scala | 56 ++++++++++++++----- .../foxcomm/search/dsl/QueryDslSpec.scala | 3 +- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index 85e37c2291..ed80d256e4 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -20,6 +20,9 @@ class SearchService(private val client: ElasticClient) extends AnyVal { val withQuery = searchQuery match { case SearchPayload.es(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces case SearchPayload.fc(query, _) ⇒ + // TODO: this is really some basic and quite ugly interpreter + // consider more principled approach + // maybe free monad would be a good fit there? (_: SearchDefinition) bool { query.query.foldLeft(new BoolQueryDefinition) { case (bool, QueryFunction.eq(in, value)) ⇒ @@ -29,6 +32,16 @@ class SearchService(private val client: ElasticClient) extends AnyVal { case (bool, QueryFunction.matches(in, value)) ⇒ val fields = in.toList bool.must(value.toList.map(q ⇒ multiMatchQuery(q).fields(fields))) + case (bool, QueryFunction.range(in, value)) ⇒ + val query = rangeQuery(in.field) + val unified = value.unify + val queryWithLowerBound = unified.lower.fold(query) { + case (b, v) ⇒ query.from(v).includeLower(b.withBound) + } + val boundedQuery = unified.upper.fold(queryWithLowerBound) { + case (b, v) ⇒ queryWithLowerBound.to(v).includeUpper(b.withBound) + } + bool.filter(boundedQuery) case (bool, _) ⇒ bool // TODO: implement rest of cases } } diff --git a/search-service/core/app/foxcomm/search/dsl/query.scala b/search-service/core/app/foxcomm/search/dsl/query.scala index 164becb21e..989763e694 100644 --- a/search-service/core/app/foxcomm/search/dsl/query.scala +++ b/search-service/core/app/foxcomm/search/dsl/query.scala @@ -41,20 +41,24 @@ object query { sealed trait RangeFunction object RangeFunction { sealed trait LowerBound extends RangeFunction { - override def hashCode(): Int = super.hashCode() - - override def equals(obj: scala.Any): Boolean = super.equals(obj) + def withBound: Boolean + } + case object Gt extends RangeFunction with LowerBound { + def withBound: Boolean = false + } + case object Gte extends RangeFunction with LowerBound { + def withBound: Boolean = true } - case object Lt extends RangeFunction with LowerBound - case object Lte extends RangeFunction with LowerBound sealed trait UpperBound extends RangeFunction { - override def hashCode(): Int = super.hashCode() - - override def equals(obj: scala.Any): Boolean = super.equals(obj) + def withBound: Boolean + } + case object Lt extends RangeFunction with UpperBound { + def withBound: Boolean = false + } + case object Lte extends RangeFunction with UpperBound { + def withBound: Boolean = true } - case object Gt extends RangeFunction with UpperBound - case object Gte extends RangeFunction with UpperBound implicit val decodeRangeFunction: KeyDecoder[RangeFunction] = KeyDecoder.instance { case "lt" | "<" ⇒ Some(Lt) @@ -73,9 +77,31 @@ object query { implicit val decodeEntityState: Decoder[EntityState] = deriveEnumerationDecoder[EntityState] } + final case class RangeBound[T](lower: Option[(RangeFunction.LowerBound, T)], + upper: Option[(RangeFunction.UpperBound, T)]) { + def toMap: Map[RangeFunction, T] = Map.empty ++ lower.toList ++ upper.toList + } + object RangeBound { + import RangeFunction._ + + implicit def decodeRangeBound[T: Decoder]: Decoder[RangeBound[T]] = + Decoder.decodeMapLike[Map, RangeFunction, T].emap { map ⇒ + val lbs = map.view.collect { + case (lb: LowerBound, v) ⇒ lb → v + }.toList + val ubs = map.view.collect { + case (ub: UpperBound, v) ⇒ ub → v + }.toList + + if (lbs.size > 1) Either.left("Only single lower bound can be specified") + else if (ubs.size > 1) Either.left("Only single upper bound can be specified") + else Either.right(RangeBound(lbs.headOption, ubs.headOption)) + } + } + type QueryValue[T] = T :+: NonEmptyList[T] :+: CNil type CompoundValue = QueryValue[JsonNumber] :+: QueryValue[String] :+: CNil - type RangeValue = Map[RangeFunction, JsonNumber] :+: Map[RangeFunction, String] :+: CNil + type RangeValue = RangeBound[JsonNumber] :+: RangeBound[String] :+: CNil object queryValueF extends Poly1 { implicit def singleValue[T] = at[T](List(_)) @@ -117,15 +143,17 @@ object query { } implicit def decodeQueryValue[T: Decoder]: Decoder[QueryValue[T]] = - Decoder[T].map(Inl(_)) or Decoder.decodeNonEmptyList[T].map(n ⇒ Inr(Inl(n))) + Decoder[T].map(Coproduct[QueryValue[T]](_)) or Decoder + .decodeNonEmptyList[T] + .map(Coproduct[QueryValue[T]](_)) implicit val decodeCompoundValue: Decoder[CompoundValue] = Decoder[QueryValue[JsonNumber]].map(Coproduct[CompoundValue](_)) or Decoder[QueryValue[String]].map(Coproduct[CompoundValue](_)) implicit val decodeRange: Decoder[RangeValue] = - Decoder.decodeMapLike[Map, RangeFunction, JsonNumber].map(Inl(_)) or - Decoder.decodeMapLike[Map, RangeFunction, String].map(sm ⇒ Inr(Inl(sm))) + Decoder[RangeBound[JsonNumber]].map(Coproduct[RangeValue](_)) or + Decoder[RangeBound[String]].map(Coproduct[RangeValue](_)) sealed trait QueryFunction object QueryFunction { diff --git a/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala b/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala index 7a86dcb75d..f8f79090c6 100644 --- a/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala +++ b/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala @@ -1,7 +1,6 @@ package foxcomm.search.dsl import foxcomm.search.dsl.query._ -import io.circe.JsonNumber import io.circe.parser._ import org.scalatest.EitherValues._ import org.scalatest.OptionValues._ @@ -32,7 +31,7 @@ class QueryDslSpec extends FlatSpec with Matchers { } assertQueryFunction[QueryFunction.range](queries(3)) { range ⇒ range.in.toList should === (List("price")) - range.value.unify.cast[Map[RangeFunction, JsonNumber]].value.mapValues(_.toString) should === ( + range.value.unify.toMap.mapValues(_.toString) should === ( Map( RangeFunction.Lt → "5000", RangeFunction.Gte → "1000" From f9229ac417887d200f16923a6333e825ae4d9663 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Wed, 14 Jun 2017 15:55:21 +0200 Subject: [PATCH 11/61] Apply small changes Allow for empty fc query, that is interpreted as `match_all` query. Return max_score with response to keep it consistent with current search result. --- .../app/foxcomm/search/SearchService.scala | 52 ++++---- .../core/app/foxcomm/search/payload.scala | 4 +- .../core/app/foxcomm/search/response.scala | 2 +- search-service/core/resources/reference.conf | 2 +- search-service/core/test/resources/es.json | 116 ------------------ search-service/core/test/resources/fc.json | 0 6 files changed, 33 insertions(+), 143 deletions(-) delete mode 100644 search-service/core/test/resources/es.json delete mode 100644 search-service/core/test/resources/fc.json diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/search-service/core/app/foxcomm/search/SearchService.scala index ed80d256e4..be9010da7a 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/search-service/core/app/foxcomm/search/SearchService.scala @@ -24,26 +24,28 @@ class SearchService(private val client: ElasticClient) extends AnyVal { // consider more principled approach // maybe free monad would be a good fit there? (_: SearchDefinition) bool { - query.query.foldLeft(new BoolQueryDefinition) { - case (bool, QueryFunction.eq(in, value)) ⇒ - bool.filter(in.toList.map(termsQuery(_, value.toList: _*))) - case (bool, QueryFunction.neq(in, value)) ⇒ - bool.not(in.toList.map(termsQuery(_, value.toList: _*))) - case (bool, QueryFunction.matches(in, value)) ⇒ - val fields = in.toList - bool.must(value.toList.map(q ⇒ multiMatchQuery(q).fields(fields))) - case (bool, QueryFunction.range(in, value)) ⇒ - val query = rangeQuery(in.field) - val unified = value.unify - val queryWithLowerBound = unified.lower.fold(query) { - case (b, v) ⇒ query.from(v).includeLower(b.withBound) - } - val boundedQuery = unified.upper.fold(queryWithLowerBound) { - case (b, v) ⇒ queryWithLowerBound.to(v).includeUpper(b.withBound) - } - bool.filter(boundedQuery) - case (bool, _) ⇒ bool // TODO: implement rest of cases - } + query + .map(_.query.foldLeft(new BoolQueryDefinition) { + case (bool, QueryFunction.eq(in, value)) ⇒ + bool.filter(in.toList.map(termsQuery(_, value.toList: _*))) + case (bool, QueryFunction.neq(in, value)) ⇒ + bool.not(in.toList.map(termsQuery(_, value.toList: _*))) + case (bool, QueryFunction.matches(in, value)) ⇒ + val fields = in.toList + bool.must(value.toList.map(q ⇒ multiMatchQuery(q).fields(fields))) + case (bool, QueryFunction.range(in, value)) ⇒ + val query = rangeQuery(in.field) + val unified = value.unify + val queryWithLowerBound = unified.lower.fold(query) { + case (b, v) ⇒ query.from(v).includeLower(b.withBound) + } + val boundedQuery = unified.upper.fold(queryWithLowerBound) { + case (b, v) ⇒ queryWithLowerBound.to(v).includeUpper(b.withBound) + } + bool.filter(boundedQuery) + case (bool, _) ⇒ bool // TODO: implement rest of cases + }) + .getOrElse((new BoolQueryDefinition).must(matchAllQuery)) } } val baseSearch = withQuery(search in searchIndex size searchSize) @@ -52,9 +54,13 @@ class SearchService(private val client: ElasticClient) extends AnyVal { client .execute(searchFrom.fold(limitedSearch)(limitedSearch from)) .map(response ⇒ - SearchResult(result = response.hits.collect { - case ExtractJsonObject(obj) ⇒ obj - }(collection.breakOut), pagination = SearchPagination(total = response.totalHits))) + SearchResult( + result = response.hits.collect { + case ExtractJsonObject(obj) ⇒ obj + }(collection.breakOut), + pagination = SearchPagination(total = response.totalHits), + maxScore = response.maxScore + )) } } diff --git a/search-service/core/app/foxcomm/search/payload.scala b/search-service/core/app/foxcomm/search/payload.scala index 36367c7bc5..c0c6a58d6e 100644 --- a/search-service/core/app/foxcomm/search/payload.scala +++ b/search-service/core/app/foxcomm/search/payload.scala @@ -8,6 +8,6 @@ sealed trait SearchPayload { def fields: Option[NonEmptyList[String]] } object SearchPayload { - final case class es(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchPayload - final case class fc(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchPayload + final case class es(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchPayload + final case class fc(query: Option[FCQuery], fields: Option[NonEmptyList[String]]) extends SearchPayload } diff --git a/search-service/core/app/foxcomm/search/response.scala b/search-service/core/app/foxcomm/search/response.scala index 345c1d0e5b..717116f8aa 100644 --- a/search-service/core/app/foxcomm/search/response.scala +++ b/search-service/core/app/foxcomm/search/response.scala @@ -2,6 +2,6 @@ package foxcomm.search import io.circe.JsonObject -final case class SearchResult(result: List[JsonObject], pagination: SearchPagination) +final case class SearchResult(result: List[JsonObject], pagination: SearchPagination, maxScore: Float) final case class SearchPagination(total: Long) diff --git a/search-service/core/resources/reference.conf b/search-service/core/resources/reference.conf index daf8c5e83f..8b16a20874 100644 --- a/search-service/core/resources/reference.conf +++ b/search-service/core/resources/reference.conf @@ -6,6 +6,6 @@ app { http { interface = "0.0.0.0" - port = 9000 + port = ${?PORT} } } diff --git a/search-service/core/test/resources/es.json b/search-service/core/test/resources/es.json deleted file mode 100644 index 4d54808fb0..0000000000 --- a/search-service/core/test/resources/es.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "query": { - "bool": { - "filter": [ - { - "term": { - "context": "default" - } - } - ], - "must": [ - { - "nested": { - "path": "taxonomies", - "query": { - "bool": { - "must": [ - { - "query": { - "bool": { - "should": { - "term": { - "taxonomies.taxons": "MEN" - } - } - } - } - } - ] - } - } - } - }, - { - "nested": { - "path": "taxonomies", - "query": { - "bool": { - "must": [ - { - "query": { - "bool": { - "should": { - "term": { - "taxonomies.taxons": "SHOES" - } - } - } - } - } - ] - } - } - } - }, - { - "nested": { - "path": "taxonomies", - "query": { - "bool": { - "must": [ - { - "query": { - "bool": { - "should": { - "term": { - "taxonomies.taxons": "OUTDOOR" - } - } - } - } - } - ] - } - } - } - } - ], - "must_not": [ - { - "term": { - "tags": "GIFT-CARD" - } - } - ] - } - }, - "sort": [ - { - "salePrice": { - "order": "asc" - } - } - ], - "aggs": { - "taxonomies": { - "nested": { - "path": "taxonomies" - }, - "aggs": { - "taxonomy": { - "terms": { - "field": "taxonomies.taxonomy" - }, - "aggs": { - "taxon": { - "terms": { - "field": "taxonomies.taxons" - } - } - } - } - } - } - } -} diff --git a/search-service/core/test/resources/fc.json b/search-service/core/test/resources/fc.json deleted file mode 100644 index e69de29bb2..0000000000 From 88dde11172a9dd132304a713f5cccee2b6f875a7 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Fri, 16 Jun 2017 11:31:15 +0200 Subject: [PATCH 12/61] Add ping endpoint --- search-service/api/app/foxcomm/search/api/Api.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/search-service/api/app/foxcomm/search/api/Api.scala b/search-service/api/app/foxcomm/search/api/Api.scala index c9d57f575a..38f6fff543 100644 --- a/search-service/api/app/foxcomm/search/api/Api.scala +++ b/search-service/api/app/foxcomm/search/api/Api.scala @@ -1,6 +1,6 @@ package foxcomm.search.api -import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.ElasticImplicits._ import com.twitter.finagle.Http import com.twitter.finagle.http.Status import com.twitter.util.Await @@ -13,7 +13,7 @@ import org.elasticsearch.common.ValidationException import scala.concurrent.ExecutionContext object Api extends App { - def endpoint(searchService: SearchService)(implicit ec: ExecutionContext) = + def endpoints(searchService: SearchService)(implicit ec: ExecutionContext) = post( "search" :: string :: string :: param("size") .as[Int] :: paramOption("from").as[Int] :: jsonBody[SearchQuery]) { @@ -22,6 +22,8 @@ object Api extends App { .searchFor(searchIndex / searchType, searchQuery, searchSize = size, searchFrom = from) .toTwitterFuture .map(Ok) + } :+: get("ping") { + Ok("pong") } def errorHandler[A]: PartialFunction[Throwable, Output[A]] = { @@ -38,5 +40,5 @@ object Api extends App { Http.server .withStreaming(enabled = true) .serve(s"${config.http.interface}:${config.http.port}", - endpoint(svc).handle(errorHandler).toServiceAs[Application.Json])) + endpoints(svc).handle(errorHandler).toServiceAs[Application.Json])) } From 8ea6b32be14a3c1088f3b59091e9973239f582d6 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Fri, 16 Jun 2017 11:39:47 +0200 Subject: [PATCH 13/61] Update CI files --- builder.py | 1 + search-service/Makefile | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/builder.py b/builder.py index 5f90a7f690..fedcaf661b 100755 --- a/builder.py +++ b/builder.py @@ -48,6 +48,7 @@ 'onboarding/ui', 'phoenix-scala', 'phoenix-scala/seeder', + 'search-service', 'solomon', 'tabernacle/docker/neo4j', 'tabernacle/docker/neo4j_reset', diff --git a/search-service/Makefile b/search-service/Makefile index e57e5f2707..e1117bff4b 100644 --- a/search-service/Makefile +++ b/search-service/Makefile @@ -8,7 +8,7 @@ SBT_CMD = sbt -DDOCKER_REPO=$(DOCKER_REPO) -DDOCKER_TAG=${DOCKER_TAG} clean: $(call header, Cleaning) - ${SBT_CMD} 'clean' + ${SBT_CMD} '; clean' build: $(call header, Building) @@ -16,15 +16,15 @@ build: test: $(call header, Testing) - ${SBT_CMD} 'test' + ${SBT_CMD} '; test' docker: $(call header, Dockerizing) - ${SBT_CMD} 'api/docker' + ${SBT_CMD} '; api/docker' docker-push: $(call header, Registering) - ${SBT_CMD} 'api/dockerPush' + ${SBT_CMD} '; api/dockerPush' -docker-build: - ${SBT_CMD} 'api/dockerBuildAndPush' +docker-all: + ${SBT_CMD} '; api/dockerBuildAndPush' From 5becf5f8b95e03d90b9aacca7d1bbe84c8bebda8 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Mon, 19 Jun 2017 12:33:07 +0200 Subject: [PATCH 14/61] Apply review suggestions --- search-service/core/app/foxcomm/search/dsl/query.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/search-service/core/app/foxcomm/search/dsl/query.scala b/search-service/core/app/foxcomm/search/dsl/query.scala index 989763e694..45a2cce0ec 100644 --- a/search-service/core/app/foxcomm/search/dsl/query.scala +++ b/search-service/core/app/foxcomm/search/dsl/query.scala @@ -104,15 +104,17 @@ object query { type RangeValue = RangeBound[JsonNumber] :+: RangeBound[String] :+: CNil object queryValueF extends Poly1 { - implicit def singleValue[T] = at[T](List(_)) + implicit def singleValue[T]: Case.Aux[T, List[T]] = at[T](List(_)) - implicit def multipleValues[T] = at[NonEmptyList[T]](_.toList) + implicit def multipleValues[T]: Case.Aux[NonEmptyList[T], List[T]] = at[NonEmptyList[T]](_.toList) } object listOfAnyValueF extends Poly1 { - implicit def caseJsonNumber = at[QueryValue[JsonNumber]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) + implicit val caseJsonNumber: Case.Aux[QueryValue[JsonNumber], List[AnyRef]] = + at[QueryValue[JsonNumber]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) - implicit def caseString = at[QueryValue[String]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) + implicit val caseString: Case.Aux[QueryValue[String], List[AnyRef]] = + at[QueryValue[String]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) } // Ugly optimisation to avoid recreation of folder instance on each implicit call From 2041ae25d7122501a6203194f849efabb5f97699 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Mon, 19 Jun 2017 14:43:37 +0200 Subject: [PATCH 15/61] Move files around --- {search-service => agni}/.gitignore | 0 {search-service => agni}/Makefile | 4 ++-- .../search => agni/api/app/foxcomm/agni}/api/Api.scala | 4 ++-- {search-service => agni}/build.sbt | 9 +++------ .../core/app/foxcomm/agni}/AppConfig.scala | 2 +- .../core/app/foxcomm/agni}/SearchService.scala | 4 ++-- agni/core/app/foxcomm/agni/dsl/package.scala | 7 +++++++ .../core/app/foxcomm/agni}/dsl/query.scala | 2 +- .../search => agni/core/app/foxcomm/agni}/package.scala | 2 +- .../search => agni/core/app/foxcomm/agni}/payload.scala | 4 ++-- .../search => agni/core/app/foxcomm/agni}/response.scala | 2 +- {search-service => agni}/core/resources/reference.conf | 0 .../core/test/foxcomm/agni}/dsl/QueryDslSpec.scala | 4 ++-- .../core/test/resources/happy_path.json | 0 .../finch/app/foxcomm/utils/finch/Conversions.scala | 0 .../finch/app/foxcomm/utils/finch/JWT.scala | 0 .../finch/app/foxcomm/utils/finch/package.scala | 0 {search-service => agni}/project/Dependencies.scala | 0 {search-service => agni}/project/Settings.scala | 0 {search-service => agni}/project/assembly.sbt | 0 {search-service => agni}/project/build.properties | 0 {search-service => agni}/project/docker.sbt | 0 {search-service => agni}/project/wartremover.sbt | 0 search-service/core/app/foxcomm/search/dsl/package.scala | 7 ------- 24 files changed, 24 insertions(+), 27 deletions(-) rename {search-service => agni}/.gitignore (100%) rename {search-service => agni}/Makefile (86%) rename {search-service/api/app/foxcomm/search => agni/api/app/foxcomm/agni}/api/Api.scala (97%) rename {search-service => agni}/build.sbt (82%) rename {search-service/core/app/foxcomm/search => agni/core/app/foxcomm/agni}/AppConfig.scala (95%) rename {search-service/core/app/foxcomm/search => agni/core/app/foxcomm/agni}/SearchService.scala (98%) create mode 100644 agni/core/app/foxcomm/agni/dsl/package.scala rename {search-service/core/app/foxcomm/search => agni/core/app/foxcomm/agni}/dsl/query.scala (99%) rename {search-service/core/app/foxcomm/search => agni/core/app/foxcomm/agni}/package.scala (88%) rename {search-service/core/app/foxcomm/search => agni/core/app/foxcomm/agni}/payload.scala (86%) rename {search-service/core/app/foxcomm/search => agni/core/app/foxcomm/agni}/response.scala (88%) rename {search-service => agni}/core/resources/reference.conf (100%) rename {search-service/core/test/foxcomm/search => agni/core/test/foxcomm/agni}/dsl/QueryDslSpec.scala (96%) rename {search-service => agni}/core/test/resources/happy_path.json (100%) rename {search-service => agni}/finch/app/foxcomm/utils/finch/Conversions.scala (100%) rename {search-service => agni}/finch/app/foxcomm/utils/finch/JWT.scala (100%) rename {search-service => agni}/finch/app/foxcomm/utils/finch/package.scala (100%) rename {search-service => agni}/project/Dependencies.scala (100%) rename {search-service => agni}/project/Settings.scala (100%) rename {search-service => agni}/project/assembly.sbt (100%) rename {search-service => agni}/project/build.properties (100%) rename {search-service => agni}/project/docker.sbt (100%) rename {search-service => agni}/project/wartremover.sbt (100%) delete mode 100644 search-service/core/app/foxcomm/search/dsl/package.scala diff --git a/search-service/.gitignore b/agni/.gitignore similarity index 100% rename from search-service/.gitignore rename to agni/.gitignore diff --git a/search-service/Makefile b/agni/Makefile similarity index 86% rename from search-service/Makefile rename to agni/Makefile index e1117bff4b..c4eddebf3d 100644 --- a/search-service/Makefile +++ b/agni/Makefile @@ -1,8 +1,8 @@ include ../makelib -header = $(call baseheader, $(1), search-service) +header = $(call baseheader, $(1), agni) DOCKER_REPO ?= $(DOCKER_STAGE_REPO) -DOCKER_IMAGE ?= search-service +DOCKER_IMAGE ?= agni DOCKER_TAG ?= master SBT_CMD = sbt -DDOCKER_REPO=$(DOCKER_REPO) -DDOCKER_TAG=${DOCKER_TAG} diff --git a/search-service/api/app/foxcomm/search/api/Api.scala b/agni/api/app/foxcomm/agni/api/Api.scala similarity index 97% rename from search-service/api/app/foxcomm/search/api/Api.scala rename to agni/api/app/foxcomm/agni/api/Api.scala index 7b7e5e4d40..e71f4de080 100644 --- a/search-service/api/app/foxcomm/search/api/Api.scala +++ b/agni/api/app/foxcomm/agni/api/Api.scala @@ -1,10 +1,10 @@ -package foxcomm.search.api +package foxcomm.agni.api import com.sksamuel.elastic4s.ElasticImplicits._ import com.twitter.finagle.Http import com.twitter.finagle.http.Status import com.twitter.util.Await -import foxcomm.search._ +import foxcomm.agni._ import foxcomm.utils.finch._ import io.circe.generic.extras.auto._ import io.finch._ diff --git a/search-service/build.sbt b/agni/build.sbt similarity index 82% rename from search-service/build.sbt rename to agni/build.sbt index 8690e41754..00528dd11b 100644 --- a/search-service/build.sbt +++ b/agni/build.sbt @@ -1,9 +1,6 @@ -import sbtassembly.AssemblyKeys.assemblyExcludedJars import sbtassembly.{MergeStrategy, PathList} -name := "search-service" - -version := "0.1-SNAPSHOT" +name := "agni" lazy val core = (project in file("core")) .settings(Settings.common) @@ -24,8 +21,8 @@ lazy val api = (project in file("api")) libraryDependencies ++= Dependencies.finch ) .settings( - mainClass in assembly := Some("foxcomm.search.api.Api"), - assemblyJarName in assembly := "search-service.jar", + mainClass in assembly := Some("foxcomm.agni.api.Api"), + assemblyJarName in assembly := s"${name.value}.jar", assemblyMergeStrategy in assembly := { case PathList("BUILD") ⇒ MergeStrategy.discard case PathList("META-INF", "io.netty.versions.properties") ⇒ MergeStrategy.discard diff --git a/search-service/core/app/foxcomm/search/AppConfig.scala b/agni/core/app/foxcomm/agni/AppConfig.scala similarity index 95% rename from search-service/core/app/foxcomm/search/AppConfig.scala rename to agni/core/app/foxcomm/agni/AppConfig.scala index 1dae9a7138..4f36ae3aec 100644 --- a/search-service/core/app/foxcomm/search/AppConfig.scala +++ b/agni/core/app/foxcomm/agni/AppConfig.scala @@ -1,4 +1,4 @@ -package foxcomm.search +package foxcomm.agni import com.typesafe.config.ConfigFactory import pureconfig._ diff --git a/search-service/core/app/foxcomm/search/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala similarity index 98% rename from search-service/core/app/foxcomm/search/SearchService.scala rename to agni/core/app/foxcomm/agni/SearchService.scala index be9010da7a..964eefc1e3 100644 --- a/search-service/core/app/foxcomm/search/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -1,10 +1,10 @@ -package foxcomm.search +package foxcomm.agni import scala.language.postfixOps import cats.implicits._ import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s._ -import foxcomm.search.dsl.query._ +import foxcomm.agni.dsl.query._ import io.circe._ import io.circe.jawn.parseByteBuffer import org.elasticsearch.common.settings.Settings diff --git a/agni/core/app/foxcomm/agni/dsl/package.scala b/agni/core/app/foxcomm/agni/dsl/package.scala new file mode 100644 index 0000000000..2aeb24ae85 --- /dev/null +++ b/agni/core/app/foxcomm/agni/dsl/package.scala @@ -0,0 +1,7 @@ +package foxcomm.agni + +import io.circe.generic.extras.Configuration + +package object dsl { + implicit def configuration: Configuration = foxcomm.agni.configuration +} diff --git a/search-service/core/app/foxcomm/search/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala similarity index 99% rename from search-service/core/app/foxcomm/search/dsl/query.scala rename to agni/core/app/foxcomm/agni/dsl/query.scala index 45a2cce0ec..eb9ad66252 100644 --- a/search-service/core/app/foxcomm/search/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -1,4 +1,4 @@ -package foxcomm.search.dsl +package foxcomm.agni.dsl import cats.data.NonEmptyList import cats.syntax.either._ diff --git a/search-service/core/app/foxcomm/search/package.scala b/agni/core/app/foxcomm/agni/package.scala similarity index 88% rename from search-service/core/app/foxcomm/search/package.scala rename to agni/core/app/foxcomm/agni/package.scala index 10e39f385e..62fc691b8e 100644 --- a/search-service/core/app/foxcomm/search/package.scala +++ b/agni/core/app/foxcomm/agni/package.scala @@ -2,7 +2,7 @@ package foxcomm import io.circe.generic.extras.Configuration -package object search { +package object agni { implicit val configuration: Configuration = Configuration.default.withDiscriminator("type").withSnakeCaseKeys } diff --git a/search-service/core/app/foxcomm/search/payload.scala b/agni/core/app/foxcomm/agni/payload.scala similarity index 86% rename from search-service/core/app/foxcomm/search/payload.scala rename to agni/core/app/foxcomm/agni/payload.scala index c0c6a58d6e..46ce769c7f 100644 --- a/search-service/core/app/foxcomm/search/payload.scala +++ b/agni/core/app/foxcomm/agni/payload.scala @@ -1,7 +1,7 @@ -package foxcomm.search +package foxcomm.agni import cats.data.NonEmptyList -import foxcomm.search.dsl.query._ +import foxcomm.agni.dsl.query._ import io.circe.JsonObject sealed trait SearchPayload { diff --git a/search-service/core/app/foxcomm/search/response.scala b/agni/core/app/foxcomm/agni/response.scala similarity index 88% rename from search-service/core/app/foxcomm/search/response.scala rename to agni/core/app/foxcomm/agni/response.scala index 717116f8aa..f385e6edbc 100644 --- a/search-service/core/app/foxcomm/search/response.scala +++ b/agni/core/app/foxcomm/agni/response.scala @@ -1,4 +1,4 @@ -package foxcomm.search +package foxcomm.agni import io.circe.JsonObject diff --git a/search-service/core/resources/reference.conf b/agni/core/resources/reference.conf similarity index 100% rename from search-service/core/resources/reference.conf rename to agni/core/resources/reference.conf diff --git a/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala similarity index 96% rename from search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala rename to agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index f8f79090c6..fd22764055 100644 --- a/search-service/core/test/foxcomm/search/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -1,6 +1,6 @@ -package foxcomm.search.dsl +package foxcomm.agni.dsl -import foxcomm.search.dsl.query._ +import foxcomm.agni.dsl.query._ import io.circe.parser._ import org.scalatest.EitherValues._ import org.scalatest.OptionValues._ diff --git a/search-service/core/test/resources/happy_path.json b/agni/core/test/resources/happy_path.json similarity index 100% rename from search-service/core/test/resources/happy_path.json rename to agni/core/test/resources/happy_path.json diff --git a/search-service/finch/app/foxcomm/utils/finch/Conversions.scala b/agni/finch/app/foxcomm/utils/finch/Conversions.scala similarity index 100% rename from search-service/finch/app/foxcomm/utils/finch/Conversions.scala rename to agni/finch/app/foxcomm/utils/finch/Conversions.scala diff --git a/search-service/finch/app/foxcomm/utils/finch/JWT.scala b/agni/finch/app/foxcomm/utils/finch/JWT.scala similarity index 100% rename from search-service/finch/app/foxcomm/utils/finch/JWT.scala rename to agni/finch/app/foxcomm/utils/finch/JWT.scala diff --git a/search-service/finch/app/foxcomm/utils/finch/package.scala b/agni/finch/app/foxcomm/utils/finch/package.scala similarity index 100% rename from search-service/finch/app/foxcomm/utils/finch/package.scala rename to agni/finch/app/foxcomm/utils/finch/package.scala diff --git a/search-service/project/Dependencies.scala b/agni/project/Dependencies.scala similarity index 100% rename from search-service/project/Dependencies.scala rename to agni/project/Dependencies.scala diff --git a/search-service/project/Settings.scala b/agni/project/Settings.scala similarity index 100% rename from search-service/project/Settings.scala rename to agni/project/Settings.scala diff --git a/search-service/project/assembly.sbt b/agni/project/assembly.sbt similarity index 100% rename from search-service/project/assembly.sbt rename to agni/project/assembly.sbt diff --git a/search-service/project/build.properties b/agni/project/build.properties similarity index 100% rename from search-service/project/build.properties rename to agni/project/build.properties diff --git a/search-service/project/docker.sbt b/agni/project/docker.sbt similarity index 100% rename from search-service/project/docker.sbt rename to agni/project/docker.sbt diff --git a/search-service/project/wartremover.sbt b/agni/project/wartremover.sbt similarity index 100% rename from search-service/project/wartremover.sbt rename to agni/project/wartremover.sbt diff --git a/search-service/core/app/foxcomm/search/dsl/package.scala b/search-service/core/app/foxcomm/search/dsl/package.scala deleted file mode 100644 index 4f382b5917..0000000000 --- a/search-service/core/app/foxcomm/search/dsl/package.scala +++ /dev/null @@ -1,7 +0,0 @@ -package foxcomm.search - -import io.circe.generic.extras.Configuration - -package object dsl { - implicit def configuration: Configuration = foxcomm.search.configuration -} From 173968bdb797c88eb8762850ee6d4c443d5e4d59 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Mon, 19 Jun 2017 14:46:39 +0200 Subject: [PATCH 16/61] Rename search service in builder.py --- builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder.py b/builder.py index fedcaf661b..a299d4a9dd 100755 --- a/builder.py +++ b/builder.py @@ -17,6 +17,7 @@ ROOT_DIR=os.path.abspath(os.path.dirname(os.path.basename(__file__))) PROJECTS = ( + 'agni', 'ashes', 'data-import', 'demo/peacock', @@ -48,7 +49,6 @@ 'onboarding/ui', 'phoenix-scala', 'phoenix-scala/seeder', - 'search-service', 'solomon', 'tabernacle/docker/neo4j', 'tabernacle/docker/neo4j_reset', From c5cf6335ddfcea232949456b2f26f2383877e917 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 20 Jun 2017 10:41:15 +0200 Subject: [PATCH 17/61] Add dsl interpreter Get rid of elastic4s and use plain java client only. Add query dsl interpreter based on tagless encoding. Use monix Coeval to handle mutability. Refactor code around. --- agni/api/app/foxcomm/agni/api/Api.scala | 19 +-- agni/build.sbt | 4 +- agni/core/app/foxcomm/agni/AppConfig.scala | 20 +++- .../core/app/foxcomm/agni/SearchService.scala | 112 +++++++++--------- agni/core/app/foxcomm/agni/dsl/query.scala | 1 - .../foxcomm/agni/interpreter/package.scala | 10 ++ .../app/foxcomm/agni/interpreter/query.scala | 79 ++++++++++++ agni/core/app/foxcomm/agni/package.scala | 20 ++++ .../test/foxcomm/agni/dsl/QueryDslSpec.scala | 7 +- agni/core/test/resources/happy_path.json | 4 - .../app/foxcomm/utils/finch/Conversions.scala | 16 +++ agni/project/Dependencies.scala | 21 ++-- 12 files changed, 232 insertions(+), 81 deletions(-) create mode 100644 agni/core/app/foxcomm/agni/interpreter/package.scala create mode 100644 agni/core/app/foxcomm/agni/interpreter/query.scala diff --git a/agni/api/app/foxcomm/agni/api/Api.scala b/agni/api/app/foxcomm/agni/api/Api.scala index e71f4de080..70d7884c59 100644 --- a/agni/api/app/foxcomm/agni/api/Api.scala +++ b/agni/api/app/foxcomm/agni/api/Api.scala @@ -1,6 +1,5 @@ package foxcomm.agni.api -import com.sksamuel.elastic4s.ElasticImplicits._ import com.twitter.finagle.Http import com.twitter.finagle.http.Status import com.twitter.util.Await @@ -9,19 +8,23 @@ import foxcomm.utils.finch._ import io.circe.generic.extras.auto._ import io.finch._ import io.finch.circe._ +import monix.execution.Scheduler import org.elasticsearch.common.ValidationException -import scala.concurrent.ExecutionContext object Api extends App { - def endpoints(searchService: SearchService)(implicit ec: ExecutionContext) = + def endpoints(searchService: SearchService)(implicit s: Scheduler) = post( "search" :: string :: string :: param("size") .as[Int] :: paramOption("from").as[Int] :: jsonBody[SearchPayload]) { (searchIndex: String, searchType: String, size: Int, from: Option[Int], searchQuery: SearchPayload) ⇒ searchService - .searchFor(searchIndex / searchType, searchQuery, searchSize = size, searchFrom = from) - .toTwitterFuture + .searchFor(searchIndex = searchIndex, + searchType = searchType, + searchQuery = searchQuery, + searchSize = size, + searchFrom = from) .map(Ok) + .toTwitterFuture } :+: get("ping") { Ok("pong") } @@ -32,9 +35,9 @@ object Api extends App { case ex ⇒ Output.failure(new RuntimeException(ex), Status.InternalServerError) } - implicit val ec: ExecutionContext = ExecutionContext.global - val config = AppConfig.load() - val svc = SearchService.fromConfig(config) + implicit val s: Scheduler = Scheduler.global + val config = AppConfig.load() + val svc = SearchService.fromConfig(config) Await.result( Http.server diff --git a/agni/build.sbt b/agni/build.sbt index 00528dd11b..53dd4b78c4 100644 --- a/agni/build.sbt +++ b/agni/build.sbt @@ -5,13 +5,13 @@ name := "agni" lazy val core = (project in file("core")) .settings(Settings.common) .settings( - libraryDependencies ++= Dependencies.core ++ Dependencies.es ++ Dependencies.circe ++ Dependencies.test.core + libraryDependencies ++= Dependencies.core ++ Dependencies.es ++ Dependencies.circe ++ Dependencies.monix ++ Dependencies.test.core ) lazy val finch = (project in file("finch")) .settings(Settings.common) .settings( - libraryDependencies ++= Dependencies.finch ++ Dependencies.circe :+ Dependencies.jwt + libraryDependencies ++= Dependencies.finch ++ Dependencies.circe ++ Dependencies.jwt ++ Dependencies.monix ) lazy val api = (project in file("api")) diff --git a/agni/core/app/foxcomm/agni/AppConfig.scala b/agni/core/app/foxcomm/agni/AppConfig.scala index 4f36ae3aec..9ee338a4a6 100644 --- a/agni/core/app/foxcomm/agni/AppConfig.scala +++ b/agni/core/app/foxcomm/agni/AppConfig.scala @@ -1,14 +1,32 @@ package foxcomm.agni +import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory +import java.net.InetSocketAddress import pureconfig._ +import scala.util.Try final case class AppConfig(http: AppConfig.Http, elasticsearch: AppConfig.ElasticSearch) +@SuppressWarnings(Array("org.wartremover.warts.Equals")) object AppConfig { + implicit val readHostConfig: ConfigReader[NonEmptyList[InetSocketAddress]] = + ConfigReader.fromNonEmptyStringTry(s ⇒ + Try { + val withoutPrefix = s.stripPrefix("elasticsearch://") + val hosts = withoutPrefix.split(',').map { host ⇒ + val parts = host.split(':') + require(parts.length == 2, + "ElasticSearch uri must be in format elasticsearch://host:port,host:port,...") + new InetSocketAddress(parts(0), parts(1).toInt) + } + require(hosts.length >= 1, "At least single ElasticSearch host should be specified") + NonEmptyList.fromListUnsafe(hosts.toList) + }) + final case class Http(interface: String, port: Int) - final case class ElasticSearch(host: String, cluster: String) + final case class ElasticSearch(host: NonEmptyList[InetSocketAddress], cluster: String) def load(): AppConfig = { val config = diff --git a/agni/core/app/foxcomm/agni/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala index 964eefc1e3..2b87fb8120 100644 --- a/agni/core/app/foxcomm/agni/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -1,84 +1,90 @@ package foxcomm.agni -import scala.language.postfixOps import cats.implicits._ -import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s._ -import foxcomm.agni.dsl.query._ +import foxcomm.agni.interpreter._ import io.circe._ import io.circe.jawn.parseByteBuffer +import monix.eval.{Coeval, Task} +import org.elasticsearch.action.search.{SearchAction, SearchRequestBuilder, SearchResponse} +import org.elasticsearch.client.Client +import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.common.settings.Settings -import scala.concurrent.{ExecutionContext, Future} +import org.elasticsearch.common.transport.InetSocketTransportAddress +import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} +import org.elasticsearch.search.SearchHit +import scala.concurrent.ExecutionContext -class SearchService(private val client: ElasticClient) extends AnyVal { +@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) +class SearchService private (client: Client)(implicit qi: QueryInterpreter[Coeval, BoolQueryBuilder]) { import SearchService.ExtractJsonObject - def searchFor(searchIndex: IndexAndTypes, + def searchFor(searchIndex: String, + searchType: String, searchQuery: SearchPayload, searchSize: Int, - searchFrom: Option[Int])(implicit ec: ExecutionContext): Future[SearchResult] = { - val withQuery = searchQuery match { - case SearchPayload.es(query, _) ⇒ (_: SearchDefinition) rawQuery Json.fromJsonObject(query).noSpaces + searchFrom: Option[Int])(implicit ec: ExecutionContext): Task[SearchResult] = { + def setupBuilder: Coeval[SearchRequestBuilder] = Coeval.eval { + val builder = new SearchRequestBuilder(client, SearchAction.INSTANCE) + builder + .setIndices(searchIndex) + .setTypes(searchType) + .setSize(searchSize) + searchFrom.foreach(builder.setFrom) + searchQuery.fields.foreach(fs ⇒ builder.setFetchSource(fs.toList.toArray, Array.empty[String])) + builder + } + + def evalQuery(builder: SearchRequestBuilder): Coeval[SearchRequestBuilder] = searchQuery match { + case SearchPayload.es(query, _) ⇒ + Coeval.eval(builder.setQuery(Printer.noSpaces.prettyByteBuffer(Json.fromJsonObject(query)).array())) case SearchPayload.fc(query, _) ⇒ - // TODO: this is really some basic and quite ugly interpreter - // consider more principled approach - // maybe free monad would be a good fit there? - (_: SearchDefinition) bool { - query - .map(_.query.foldLeft(new BoolQueryDefinition) { - case (bool, QueryFunction.eq(in, value)) ⇒ - bool.filter(in.toList.map(termsQuery(_, value.toList: _*))) - case (bool, QueryFunction.neq(in, value)) ⇒ - bool.not(in.toList.map(termsQuery(_, value.toList: _*))) - case (bool, QueryFunction.matches(in, value)) ⇒ - val fields = in.toList - bool.must(value.toList.map(q ⇒ multiMatchQuery(q).fields(fields))) - case (bool, QueryFunction.range(in, value)) ⇒ - val query = rangeQuery(in.field) - val unified = value.unify - val queryWithLowerBound = unified.lower.fold(query) { - case (b, v) ⇒ query.from(v).includeLower(b.withBound) - } - val boundedQuery = unified.upper.fold(queryWithLowerBound) { - case (b, v) ⇒ queryWithLowerBound.to(v).includeUpper(b.withBound) - } - bool.filter(boundedQuery) - case (bool, _) ⇒ bool // TODO: implement rest of cases - }) - .getOrElse((new BoolQueryDefinition).must(matchAllQuery)) + query.fold { + Coeval.eval(builder.setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()))) + } { q ⇒ + q.query.foldM(QueryBuilders.boolQuery())(qi.eval).map(builder.setQuery) } } - val baseSearch = withQuery(search in searchIndex size searchSize) - val limitedSearch = - searchQuery.fields.fold(baseSearch)(fields ⇒ baseSearch sourceInclude (fields.toList: _*)) - client - .execute(searchFrom.fold(limitedSearch)(limitedSearch from)) - .map(response ⇒ - SearchResult( - result = response.hits.collect { + + for { + builder ← setupBuilder.flatMap(evalQuery).task + request = builder.request() + response ← async[SearchResponse, SearchResult](client.search(request, _)) + } yield { + val hits = response.getHits + SearchResult( + result = hits + .hits() + .view + .collect { case ExtractJsonObject(obj) ⇒ obj - }(collection.breakOut), - pagination = SearchPagination(total = response.totalHits), - maxScore = response.maxScore - )) + } + .toList, + pagination = SearchPagination(total = hits.totalHits()), + maxScore = hits.getMaxScore + ) + } } } object SearchService { object ExtractJsonObject { - def unapply(hit: RichSearchHit): Option[JsonObject] = + def unapply(hit: SearchHit): Option[JsonObject] = parseByteBuffer(hit.sourceRef.toChannelBuffer.toByteBuffer).toOption .flatMap(_.asObject) } - def apply(client: ElasticClient): SearchService = new SearchService(client) + def apply(client: Client)(implicit qi: QueryInterpreter[Coeval, BoolQueryBuilder]): SearchService = + new SearchService(client) - def fromConfig(config: AppConfig): SearchService = { + def fromConfig(config: AppConfig)(implicit qi: QueryInterpreter[Coeval, BoolQueryBuilder]): SearchService = { val esConfig = config.elasticsearch val settings = Settings.settingsBuilder().put("cluster.name", esConfig.cluster).build() - val client = - ElasticClient.transport(settings, ElasticsearchClientUri(esConfig.host)) + val client = TransportClient + .builder() + .settings(settings) + .build() + .addTransportAddresses(esConfig.host.toList.map(new InetSocketTransportAddress(_)): _*) new SearchService(client) } diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index eb9ad66252..44544e3e86 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -163,7 +163,6 @@ object query { final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction final case class eq(in: QueryField, value: CompoundValue) extends QueryFunction final case class neq(in: QueryField, value: CompoundValue) extends QueryFunction - final case class state(value: EntityState) extends QueryFunction implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } diff --git a/agni/core/app/foxcomm/agni/interpreter/package.scala b/agni/core/app/foxcomm/agni/interpreter/package.scala new file mode 100644 index 0000000000..b463d96039 --- /dev/null +++ b/agni/core/app/foxcomm/agni/interpreter/package.scala @@ -0,0 +1,10 @@ +package foxcomm.agni + +import cats.~> +import monix.eval.{Coeval, Task} + +package object interpreter { + implicit val coevalToTask: Coeval ~> Task = new (Coeval ~> Task) { + def apply[A](fa: Coeval[A]): Task[A] = fa.task + } +} diff --git a/agni/core/app/foxcomm/agni/interpreter/query.scala b/agni/core/app/foxcomm/agni/interpreter/query.scala new file mode 100644 index 0000000000..79185897f3 --- /dev/null +++ b/agni/core/app/foxcomm/agni/interpreter/query.scala @@ -0,0 +1,79 @@ +package foxcomm.agni.interpreter + +import scala.language.higherKinds +import cats.implicits._ +import cats.~> +import foxcomm.agni.dsl.query._ +import monix.eval.Coeval +import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} + +sealed trait QueryInterpreter[F[_], V] { + final def eval(v: V, qf: QueryFunction): F[V] = qf match { + case qf: QueryFunction.matches ⇒ matchesF(v, qf) + case qf: QueryFunction.range ⇒ rangeF(v, qf) + case qf: QueryFunction.eq ⇒ eqF(v, qf) + case qf: QueryFunction.neq ⇒ neqF(v, qf) + } + + def matchesF(v: V, qf: QueryFunction.matches): F[V] + + def rangeF(v: V, qf: QueryFunction.range): F[V] + + def eqF(v: V, qf: QueryFunction.eq): F[V] + + def neqF(v: V, f: QueryFunction.neq): F[V] +} + +trait LowPriorityQueryInterpreters { + @inline implicit def defaultQueryInterpreter: QueryInterpreter[Coeval, BoolQueryBuilder] = + DefaultQueryInterpreter +} + +object QueryInterpreter extends LowPriorityQueryInterpreters { + @inline implicit def apply[F[_], V](implicit qi: QueryInterpreter[F, V]): QueryInterpreter[F, V] = qi + + implicit class RichQueryInterpreter[F1[_], V](private val qi: QueryInterpreter[F1, V]) extends AnyVal { + def mapTo[F2[_]](implicit nat: F1 ~> F2): QueryInterpreter[F2, V] = new QueryInterpreter[F2, V] { + def matchesF(v: V, qf: QueryFunction.matches): F2[V] = nat(qi.matchesF(v, qf)) + + def rangeF(v: V, qf: QueryFunction.range): F2[V] = nat(qi.rangeF(v, qf)) + + def eqF(v: V, qf: QueryFunction.eq): F2[V] = nat(qi.eqF(v, qf)) + + def neqF(v: V, qf: QueryFunction.neq): F2[V] = nat(qi.neqF(v, qf)) + } + } +} + +object DefaultQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { + val empty: Coeval[BoolQueryBuilder] = Coeval.eval(QueryBuilders.boolQuery()) + + def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = Coeval.evalOnce { + val fields = qf.in.toList + qf.value.toList.foldLeft(b)((b, v) ⇒ b.must(QueryBuilders.multiMatchQuery(v, fields: _*))) + } + + def rangeF(b: BoolQueryBuilder, qf: QueryFunction.range): Coeval[BoolQueryBuilder] = Coeval.evalOnce { + val builder = QueryBuilders.rangeQuery(qf.in.field) + val value = qf.value.unify + value.lower.foreach { + case (RangeFunction.Gt, v) ⇒ builder.gt(v) + case (RangeFunction.Gte, v) ⇒ builder.gte(v) + } + value.upper.foreach { + case (RangeFunction.Lt, v) ⇒ builder.lt(v) + case (RangeFunction.Lte, v) ⇒ builder.lte(v) + } + b.filter(builder) + } + + def eqF(b: BoolQueryBuilder, qf: QueryFunction.eq): Coeval[BoolQueryBuilder] = Coeval.evalOnce { + val values = qf.value.toList + qf.in.toList.foldLeft(b)((b, n) ⇒ b.filter(QueryBuilders.termsQuery(n, values: _*))) + } + + def neqF(b: BoolQueryBuilder, qf: QueryFunction.neq): Coeval[BoolQueryBuilder] = Coeval.evalOnce { + val values = qf.value.toList + qf.in.toList.foldLeft(b)((b, n) ⇒ b.mustNot(QueryBuilders.termsQuery(n, values: _*))) + } +} diff --git a/agni/core/app/foxcomm/agni/package.scala b/agni/core/app/foxcomm/agni/package.scala index 62fc691b8e..f234a9653b 100644 --- a/agni/core/app/foxcomm/agni/package.scala +++ b/agni/core/app/foxcomm/agni/package.scala @@ -1,8 +1,28 @@ package foxcomm +import cats.Monad import io.circe.generic.extras.Configuration +import monix.cats._ +import monix.eval.{Coeval, Task} +import org.elasticsearch.action.ActionListener +import scala.concurrent.Promise package object agni { implicit val configuration: Configuration = Configuration.default.withDiscriminator("type").withSnakeCaseKeys + + // FIXME: For now we cache converted monix to cats monad in order to save unnecessary allocation on every call. + // Should be ready to remove once monix 3.0.x will become stable. + implicit val coevalMonad: Monad[Coeval] = monixToCatsMonad[Coeval] + + @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) + def async[A, B](action: ActionListener[A] ⇒ Any): Task[A] = Task.deferFuture { + val p = Promise[A]() + action(new ActionListener[A] { + def onFailure(e: Throwable): Unit = p.tryFailure(e) + + def onResponse(response: A): Unit = p.trySuccess(response) + }) + p.future + } } diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index fd22764055..8ae70dfada 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -22,14 +22,11 @@ class QueryDslSpec extends FlatSpec with Matchers { is.in.toList should === (List("slug")) is.value.toList should === (List("awesome", "whatever")) } - assertQueryFunction[QueryFunction.state](queries(1)) { state ⇒ - state.value should === (EntityState.all) - } - assertQueryFunction[QueryFunction.matches](queries(2)) { matches ⇒ + assertQueryFunction[QueryFunction.matches](queries(1)) { matches ⇒ matches.in.toList should === (List("title", "description")) matches.value.toList should === (List("food", "drink")) } - assertQueryFunction[QueryFunction.range](queries(3)) { range ⇒ + assertQueryFunction[QueryFunction.range](queries(2)) { range ⇒ range.in.toList should === (List("price")) range.value.unify.toMap.mapValues(_.toString) should === ( Map( diff --git a/agni/core/test/resources/happy_path.json b/agni/core/test/resources/happy_path.json index 0a425ed9a8..630a71e829 100644 --- a/agni/core/test/resources/happy_path.json +++ b/agni/core/test/resources/happy_path.json @@ -4,10 +4,6 @@ "in": "slug", "value": [ "awesome", "whatever" ] }, - { - "type": "state", - "value": "all" - }, { "type": "matches", "in": [ "title", "description" ], diff --git a/agni/finch/app/foxcomm/utils/finch/Conversions.scala b/agni/finch/app/foxcomm/utils/finch/Conversions.scala index 0952292e97..11ca5ebfef 100644 --- a/agni/finch/app/foxcomm/utils/finch/Conversions.scala +++ b/agni/finch/app/foxcomm/utils/finch/Conversions.scala @@ -5,9 +5,13 @@ import com.twitter.util.{Return, Throw, Future ⇒ TFuture, Promise ⇒ TPromise import scala.concurrent.{ExecutionContext, Future ⇒ SFuture, Promise ⇒ SPromise} import scala.util.{Failure, Success} import Conversions._ +import monix.eval.{Callback, Task} +import monix.execution.Scheduler @SuppressWarnings(Array("org.wartremover.warts.ImplicitConversion")) trait Conversions { + implicit def toRichTask[A](task: Task[A]): RichTask[A] = new RichTask(task) + implicit def toRichSFuture[A](future: SFuture[A]): RichSFuture[A] = new RichSFuture(future) implicit def toRichTFuture[A](future: TFuture[A]): RichTFuture[A] = new RichTFuture(future) @@ -15,6 +19,18 @@ trait Conversions { @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) object Conversions { + implicit class RichTask[A](val task: Task[A]) extends AnyVal { + def toTwitterFuture(implicit s: Scheduler): TFuture[A] = { + val result = TPromise[A]() + task.runAsync(new Callback[A] { + def onError(ex: Throwable): Unit = result.setException(ex) + + def onSuccess(value: A): Unit = result.setValue(value) + }) + result + } + } + implicit class RichSFuture[A](val future: SFuture[A]) extends AnyVal { def toTwitterFuture(implicit ec: ExecutionContext): TFuture[A] = { val result = TPromise[A]() diff --git a/agni/project/Dependencies.scala b/agni/project/Dependencies.scala index 8600abe3c7..5ed2d15592 100644 --- a/agni/project/Dependencies.scala +++ b/agni/project/Dependencies.scala @@ -2,10 +2,11 @@ import sbt._ object Dependencies { object versions { - val cats = "0.9.0" - val circe = "0.8.0" - val elastic4s = "2.1.2" - val finch = "0.14.0" + val cats = "0.9.0" + val circe = "0.8.0" + val elasticsearch = "2.1.2" + val finch = "0.14.0" + val monix = "2.3.0" } val core = Seq( @@ -15,8 +16,7 @@ object Dependencies { ) val es = Seq( - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-smile" % "2.8.2", - "com.sksamuel.elastic4s" %% "elastic4s-core" % versions.elastic4s + "org.elasticsearch" % "elasticsearch" % versions.elasticsearch ) val circe = Seq( @@ -31,7 +31,14 @@ object Dependencies { "com.github.finagle" %% "finch-generic" % versions.finch ) - val jwt = "com.pauldijou" %% "jwt-core" % "0.12.1" + val jwt = Seq( + "com.pauldijou" %% "jwt-core" % "0.12.1" + ) + + val monix = Seq( + "io.monix" %% "monix-cats" % versions.monix, + "io.monix" %% "monix-eval" % versions.monix + ) object test { def core = From eb4f8d3919f43b7882424d900214ae860fd697a0 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 20 Jun 2017 10:52:47 +0200 Subject: [PATCH 18/61] Fix image name --- agni/build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agni/build.sbt b/agni/build.sbt index 00528dd11b..ebb9d58b29 100644 --- a/agni/build.sbt +++ b/agni/build.sbt @@ -22,7 +22,7 @@ lazy val api = (project in file("api")) ) .settings( mainClass in assembly := Some("foxcomm.agni.api.Api"), - assemblyJarName in assembly := s"${name.value}.jar", + assemblyJarName in assembly := s"agni.jar", assemblyMergeStrategy in assembly := { case PathList("BUILD") ⇒ MergeStrategy.discard case PathList("META-INF", "io.netty.versions.properties") ⇒ MergeStrategy.discard From dc0f94ba33ca0a7ce15733be52eb264416527fc8 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 20 Jun 2017 14:12:02 +0200 Subject: [PATCH 19/61] Make query interpreter extending function --- agni/core/app/foxcomm/agni/SearchService.scala | 2 +- agni/core/app/foxcomm/agni/interpreter/query.scala | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/agni/core/app/foxcomm/agni/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala index 2b87fb8120..17cd235dca 100644 --- a/agni/core/app/foxcomm/agni/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -41,7 +41,7 @@ class SearchService private (client: Client)(implicit qi: QueryInterpreter[Coeva query.fold { Coeval.eval(builder.setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()))) } { q ⇒ - q.query.foldM(QueryBuilders.boolQuery())(qi.eval).map(builder.setQuery) + qi(QueryBuilders.boolQuery(), q.query).map(builder.setQuery) } } diff --git a/agni/core/app/foxcomm/agni/interpreter/query.scala b/agni/core/app/foxcomm/agni/interpreter/query.scala index 79185897f3..0ffe5622c8 100644 --- a/agni/core/app/foxcomm/agni/interpreter/query.scala +++ b/agni/core/app/foxcomm/agni/interpreter/query.scala @@ -1,13 +1,15 @@ package foxcomm.agni.interpreter import scala.language.higherKinds +import cats.data.NonEmptyList import cats.implicits._ -import cats.~> +import cats.{~>, Monad} +import foxcomm.agni._ import foxcomm.agni.dsl.query._ import monix.eval.Coeval import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} -sealed trait QueryInterpreter[F[_], V] { +sealed abstract class QueryInterpreter[F[_]: Monad, V] extends ((V, NonEmptyList[QueryFunction]) ⇒ F[V]) { final def eval(v: V, qf: QueryFunction): F[V] = qf match { case qf: QueryFunction.matches ⇒ matchesF(v, qf) case qf: QueryFunction.range ⇒ rangeF(v, qf) @@ -15,13 +17,15 @@ sealed trait QueryInterpreter[F[_], V] { case qf: QueryFunction.neq ⇒ neqF(v, qf) } + final def apply(v: V, qfs: NonEmptyList[QueryFunction]): F[V] = qfs.foldM(v)(eval) + def matchesF(v: V, qf: QueryFunction.matches): F[V] def rangeF(v: V, qf: QueryFunction.range): F[V] def eqF(v: V, qf: QueryFunction.eq): F[V] - def neqF(v: V, f: QueryFunction.neq): F[V] + def neqF(v: V, qf: QueryFunction.neq): F[V] } trait LowPriorityQueryInterpreters { @@ -33,7 +37,7 @@ object QueryInterpreter extends LowPriorityQueryInterpreters { @inline implicit def apply[F[_], V](implicit qi: QueryInterpreter[F, V]): QueryInterpreter[F, V] = qi implicit class RichQueryInterpreter[F1[_], V](private val qi: QueryInterpreter[F1, V]) extends AnyVal { - def mapTo[F2[_]](implicit nat: F1 ~> F2): QueryInterpreter[F2, V] = new QueryInterpreter[F2, V] { + def mapTo[F2[_]: Monad](implicit nat: F1 ~> F2): QueryInterpreter[F2, V] = new QueryInterpreter[F2, V] { def matchesF(v: V, qf: QueryFunction.matches): F2[V] = nat(qi.matchesF(v, qf)) def rangeF(v: V, qf: QueryFunction.range): F2[V] = nat(qi.rangeF(v, qf)) @@ -46,8 +50,6 @@ object QueryInterpreter extends LowPriorityQueryInterpreters { } object DefaultQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { - val empty: Coeval[BoolQueryBuilder] = Coeval.eval(QueryBuilders.boolQuery()) - def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = Coeval.evalOnce { val fields = qf.in.toList qf.value.toList.foldLeft(b)((b, v) ⇒ b.must(QueryBuilders.multiMatchQuery(v, fields: _*))) From b1725f4875b016a5b002f6d9c1e1bd33d7e04bff Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 20 Jun 2017 15:51:02 +0200 Subject: [PATCH 20/61] Add JAVA_OPTS to search service docker image cmd --- agni/build.sbt | 5 ++--- agni/project/Settings.scala | 6 ++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/agni/build.sbt b/agni/build.sbt index ebb9d58b29..8c7c2e4a17 100644 --- a/agni/build.sbt +++ b/agni/build.sbt @@ -1,7 +1,5 @@ import sbtassembly.{MergeStrategy, PathList} -name := "agni" - lazy val core = (project in file("core")) .settings(Settings.common) .settings( @@ -21,8 +19,9 @@ lazy val api = (project in file("api")) libraryDependencies ++= Dependencies.finch ) .settings( + Settings.appName := "agni", mainClass in assembly := Some("foxcomm.agni.api.Api"), - assemblyJarName in assembly := s"agni.jar", + assemblyJarName in assembly := s"${Settings.appName.value}.jar", assemblyMergeStrategy in assembly := { case PathList("BUILD") ⇒ MergeStrategy.discard case PathList("META-INF", "io.netty.versions.properties") ⇒ MergeStrategy.discard diff --git a/agni/project/Settings.scala b/agni/project/Settings.scala index 4609602ac4..4074820816 100644 --- a/agni/project/Settings.scala +++ b/agni/project/Settings.scala @@ -9,6 +9,8 @@ import sbtdocker.immutable.Dockerfile import wartremover.{wartremoverErrors, Wart, Warts} object Settings { + lazy val appName: TaskKey[String] = taskKey[String]("Name for a application") + def common: Seq[Def.Setting[_]] = Seq( scalaVersion := "2.11.11", scalacOptions in Compile ++= Seq( @@ -47,11 +49,11 @@ object Settings { Dockerfile.empty .from("openjdk:8-alpine") .add(artifact, artifactTargetPath) - .cmdRaw(s"java -jar $artifactTargetPath") + .cmdRaw(s"java $$JAVA_OPTS -jar $artifactTargetPath 2>&1 | tee /logs/${appName.value}.log") }, imageNames in docker := Seq( ImageName( - s"${sys.props("DOCKER_REPO")}/${(assemblyJarName in assembly).value.stripSuffix(".jar")}:${sys.props("DOCKER_TAG")}") + s"${sys.props("DOCKER_REPO")}/${appName.value}:${sys.props("DOCKER_TAG")}") ) ) } From 9f614ba5cf08d467a5234a458ec9253e99dfa628 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 20 Jun 2017 16:13:31 +0200 Subject: [PATCH 21/61] Apply PR suggestions --- agni/project/Settings.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agni/project/Settings.scala b/agni/project/Settings.scala index 4074820816..c1f080fc2f 100644 --- a/agni/project/Settings.scala +++ b/agni/project/Settings.scala @@ -49,7 +49,7 @@ object Settings { Dockerfile.empty .from("openjdk:8-alpine") .add(artifact, artifactTargetPath) - .cmdRaw(s"java $$JAVA_OPTS -jar $artifactTargetPath 2>&1 | tee /logs/${appName.value}.log") + .cmdRaw(s"java $$JAVA_OPTS -jar $artifactTargetPath 2>&1 | tee -a /logs/${appName.value}.log") }, imageNames in docker := Seq( ImageName( From 9c778c86067f2094df6711d744de743357251722 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 20 Jun 2017 17:57:17 +0200 Subject: [PATCH 22/61] Bring back jackson dependency --- agni/project/Dependencies.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agni/project/Dependencies.scala b/agni/project/Dependencies.scala index 5ed2d15592..715a1436b4 100644 --- a/agni/project/Dependencies.scala +++ b/agni/project/Dependencies.scala @@ -16,7 +16,8 @@ object Dependencies { ) val es = Seq( - "org.elasticsearch" % "elasticsearch" % versions.elasticsearch + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-smile" % "2.8.2", + "org.elasticsearch" % "elasticsearch" % versions.elasticsearch ) val circe = Seq( From e9e0d31f1c008af2a5790e7a4ad80dc44e08c0bf Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Wed, 21 Jun 2017 16:23:39 +0200 Subject: [PATCH 23/61] Make QueryInterpreter extend function Gain arrow and function composition by making interpreter single-arg function. Make query functions context-aware. Refactor code around. --- agni/api/app/foxcomm/agni/api/Api.scala | 2 +- .../core/app/foxcomm/agni/SearchService.scala | 25 +++--- agni/core/app/foxcomm/agni/dsl/query.scala | 59 ++++++++++---- .../agni/interpreter/QueryInterpreter.scala | 30 +++++++ .../interpreter/es/ESQueryInterpreter.scala | 69 ++++++++++++++++ .../foxcomm/agni/interpreter/es/package.scala | 13 +++ .../foxcomm/agni/interpreter/package.scala | 7 +- .../app/foxcomm/agni/interpreter/query.scala | 81 ------------------- agni/core/app/foxcomm/agni/package.scala | 10 +-- 9 files changed, 174 insertions(+), 122 deletions(-) create mode 100644 agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala create mode 100644 agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala create mode 100644 agni/core/app/foxcomm/agni/interpreter/es/package.scala delete mode 100644 agni/core/app/foxcomm/agni/interpreter/query.scala diff --git a/agni/api/app/foxcomm/agni/api/Api.scala b/agni/api/app/foxcomm/agni/api/Api.scala index 70d7884c59..895e26992d 100644 --- a/agni/api/app/foxcomm/agni/api/Api.scala +++ b/agni/api/app/foxcomm/agni/api/Api.scala @@ -37,7 +37,7 @@ object Api extends App { implicit val s: Scheduler = Scheduler.global val config = AppConfig.load() - val svc = SearchService.fromConfig(config) + val svc = SearchService.fromConfig(config, interpreter.es.default) Await.result( Http.server diff --git a/agni/core/app/foxcomm/agni/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala index 17cd235dca..cbc03c1442 100644 --- a/agni/core/app/foxcomm/agni/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -1,7 +1,7 @@ package foxcomm.agni import cats.implicits._ -import foxcomm.agni.interpreter._ +import foxcomm.agni.interpreter.es._ import io.circe._ import io.circe.jawn.parseByteBuffer import monix.eval.{Coeval, Task} @@ -10,20 +10,19 @@ import org.elasticsearch.client.Client import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.transport.InetSocketTransportAddress -import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.SearchHit -import scala.concurrent.ExecutionContext @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) -class SearchService private (client: Client)(implicit qi: QueryInterpreter[Coeval, BoolQueryBuilder]) { +class SearchService private (client: Client, qi: ESQueryInterpreter) { import SearchService.ExtractJsonObject def searchFor(searchIndex: String, searchType: String, searchQuery: SearchPayload, searchSize: Int, - searchFrom: Option[Int])(implicit ec: ExecutionContext): Task[SearchResult] = { - def setupBuilder: Coeval[SearchRequestBuilder] = Coeval.eval { + searchFrom: Option[Int]): Task[SearchResult] = { + def prepareBuilder: Coeval[SearchRequestBuilder] = Coeval.eval { val builder = new SearchRequestBuilder(client, SearchAction.INSTANCE) builder .setIndices(searchIndex) @@ -41,12 +40,14 @@ class SearchService private (client: Client)(implicit qi: QueryInterpreter[Coeva query.fold { Coeval.eval(builder.setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()))) } { q ⇒ - qi(QueryBuilders.boolQuery(), q.query).map(builder.setQuery) + qi(q.query).map(builder.setQuery) } } + def setupBuilder: Task[SearchRequestBuilder] = (prepareBuilder flatMap evalQuery).task + for { - builder ← setupBuilder.flatMap(evalQuery).task + builder ← setupBuilder request = builder.request() response ← async[SearchResponse, SearchResult](client.search(request, _)) } yield { @@ -73,10 +74,10 @@ object SearchService { .flatMap(_.asObject) } - def apply(client: Client)(implicit qi: QueryInterpreter[Coeval, BoolQueryBuilder]): SearchService = - new SearchService(client) + def apply(client: Client, qi: ESQueryInterpreter): SearchService = + new SearchService(client, qi) - def fromConfig(config: AppConfig)(implicit qi: QueryInterpreter[Coeval, BoolQueryBuilder]): SearchService = { + def fromConfig(config: AppConfig, qi: ESQueryInterpreter): SearchService = { val esConfig = config.elasticsearch val settings = Settings.settingsBuilder().put("cluster.name", esConfig.cluster).build() @@ -86,6 +87,6 @@ object SearchService { .build() .addTransportAddresses(esConfig.host.toList.map(new InetSocketTransportAddress(_)): _*) - new SearchService(client) + apply(client, qi) } } diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 44544e3e86..b5ed461af7 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -7,7 +7,10 @@ import io.circe.{Decoder, JsonNumber, KeyDecoder} import shapeless._ import shapeless.ops.coproduct.Folder -@SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.ExplicitImplicitTypes")) +@SuppressWarnings( + Array("org.wartremover.warts.AsInstanceOf", + "org.wartremover.warts.ExplicitImplicitTypes", + "org.wartremover.warts.DefaultArguments")) object query { sealed trait QueryField { def toList: List[String] @@ -38,6 +41,16 @@ object query { Decoder[Multiple].map(m ⇒ m: QueryField) } + sealed trait QueryContext + object QueryContext { + case object filter extends QueryContext + case object must extends QueryContext + case object should extends QueryContext + case object not extends QueryContext + + implicit val decodeQueryContext: Decoder[QueryContext] = deriveEnumerationDecoder[QueryContext] + } + sealed trait RangeFunction object RangeFunction { sealed trait LowerBound extends RangeFunction { @@ -68,15 +81,6 @@ object query { } } - sealed trait EntityState - object EntityState { - case object all extends EntityState - case object active extends EntityState - case object inactive extends EntityState - - implicit val decodeEntityState: Decoder[EntityState] = deriveEnumerationDecoder[EntityState] - } - final case class RangeBound[T](lower: Option[(RangeFunction.LowerBound, T)], upper: Option[(RangeFunction.UpperBound, T)]) { def toMap: Map[RangeFunction, T] = Map.empty ++ lower.toList ++ upper.toList @@ -159,10 +163,37 @@ object query { sealed trait QueryFunction object QueryFunction { - final case class matches(in: QueryField, value: QueryValue[String]) extends QueryFunction - final case class range(in: QueryField.Single, value: RangeValue) extends QueryFunction - final case class eq(in: QueryField, value: CompoundValue) extends QueryFunction - final case class neq(in: QueryField, value: CompoundValue) extends QueryFunction + sealed trait WithField { this: QueryFunction ⇒ + def in: QueryField + } + sealed trait WithContext { this: QueryFunction ⇒ + def context: QueryContext + } + + final case class matches private (in: QueryField, + value: QueryValue[String], + context: QueryContext = QueryContext.must) + extends QueryFunction + with WithContext + with WithField + final case class range private (in: QueryField.Single, + value: RangeValue, + context: QueryContext = QueryContext.filter) + extends QueryFunction + with WithContext + with WithField + final case class eq private (in: QueryField, + value: CompoundValue, + context: QueryContext = QueryContext.filter) + extends QueryFunction + with WithContext + with WithField + final case class neq private (in: QueryField, + value: CompoundValue, + context: QueryContext = QueryContext.not) + extends QueryFunction + with WithContext + with WithField implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } diff --git a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala new file mode 100644 index 0000000000..a82bec9e73 --- /dev/null +++ b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala @@ -0,0 +1,30 @@ +package foxcomm.agni.interpreter + +import scala.language.higherKinds +import cats.Monad +import cats.data.NonEmptyList +import cats.implicits._ +import foxcomm.agni.dsl.query._ + +abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[(V, NonEmptyList[QueryFunction]), F[V]] { + final def eval(v: V, qf: QueryFunction): F[V] = qf match { + case qf: QueryFunction.matches ⇒ matchesF(v, qf) + case qf: QueryFunction.range ⇒ rangeF(v, qf) + case qf: QueryFunction.eq ⇒ eqF(v, qf) + case qf: QueryFunction.neq ⇒ neqF(v, qf) + } + + final def apply(v: (V, NonEmptyList[QueryFunction])): F[V] = v._2.foldM(v._1)(eval) + + def matchesF(v: V, qf: QueryFunction.matches): F[V] + + def rangeF(v: V, qf: QueryFunction.range): F[V] + + def eqF(v: V, qf: QueryFunction.eq): F[V] + + def neqF(v: V, qf: QueryFunction.neq): F[V] +} + +object QueryInterpreter { + @inline implicit def apply[F[_], V](implicit qi: QueryInterpreter[F, V]): QueryInterpreter[F, V] = qi +} diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala new file mode 100644 index 0000000000..e17af26c15 --- /dev/null +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -0,0 +1,69 @@ +package foxcomm.agni.interpreter.es + +import foxcomm.agni.dsl.query._ +import foxcomm.agni.interpreter.QueryInterpreter +import monix.cats._ +import monix.eval.Coeval +import org.elasticsearch.index.query._ + +@SuppressWarnings(Array("org.wartremover.warts.Overloading")) +object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { + implicit class RichBoolQueryBuilder(val b: BoolQueryBuilder) extends AnyVal { + def inContext(qf: QueryFunction.WithContext)(qb: ⇒ QueryBuilder): BoolQueryBuilder = qf.context match { + case QueryContext.filter ⇒ b.filter(qb) + case QueryContext.must ⇒ b.must(qb) + case QueryContext.should ⇒ b.should(qb) + case QueryContext.not ⇒ b.mustNot(qb) + } + + def foreachField(qf: QueryFunction.WithField)( + f: (BoolQueryBuilder, String) ⇒ BoolQueryBuilder): BoolQueryBuilder = { + qf.in.toList.foreach(f(b, _)) + b + } + + def foreachField(qf: QueryFunction.WithContext with QueryFunction.WithField)( + f: String ⇒ QueryBuilder): BoolQueryBuilder = { + qf.in.toList.foreach(n ⇒ b.inContext(qf)(f(n))) + b + } + } + + def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = Coeval.eval { + val fields = qf.in.toList + qf.value.toList.foldLeft(b)((b, v) ⇒ + b.inContext(qf) { + QueryBuilders.multiMatchQuery(v, fields: _*) + }) + } + + def rangeF(b: BoolQueryBuilder, qf: QueryFunction.range): Coeval[BoolQueryBuilder] = Coeval.eval { + b.inContext(qf) { + val builder = QueryBuilders.rangeQuery(qf.in.field) + val value = qf.value.unify + value.lower.foreach { + case (RangeFunction.Gt, v) ⇒ builder.gt(v) + case (RangeFunction.Gte, v) ⇒ builder.gte(v) + } + value.upper.foreach { + case (RangeFunction.Lt, v) ⇒ builder.lt(v) + case (RangeFunction.Lte, v) ⇒ builder.lte(v) + } + builder + } + } + + def eqF(b: BoolQueryBuilder, qf: QueryFunction.eq): Coeval[BoolQueryBuilder] = Coeval.eval { + val values = qf.value.toList + b.foreachField(qf) { n ⇒ + QueryBuilders.termsQuery(n, values: _*) + } + } + + def neqF(b: BoolQueryBuilder, qf: QueryFunction.neq): Coeval[BoolQueryBuilder] = Coeval.eval { + val values = qf.value.toList + b.foreachField(qf) { n ⇒ + QueryBuilders.termsQuery(n, values: _*) + } + } +} diff --git a/agni/core/app/foxcomm/agni/interpreter/es/package.scala b/agni/core/app/foxcomm/agni/interpreter/es/package.scala new file mode 100644 index 0000000000..d97febf3c9 --- /dev/null +++ b/agni/core/app/foxcomm/agni/interpreter/es/package.scala @@ -0,0 +1,13 @@ +package foxcomm.agni.interpreter + +import cats.data.NonEmptyList +import cats.implicits._ +import foxcomm.agni.dsl.query.QueryFunction +import monix.eval.Coeval +import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} + +package object es { + type ESQueryInterpreter = Interpreter[NonEmptyList[QueryFunction], Coeval[BoolQueryBuilder]] + + lazy val default: ESQueryInterpreter = ESQueryInterpreter <<< (QueryBuilders.boolQuery() → _) +} diff --git a/agni/core/app/foxcomm/agni/interpreter/package.scala b/agni/core/app/foxcomm/agni/interpreter/package.scala index b463d96039..aae22a9f79 100644 --- a/agni/core/app/foxcomm/agni/interpreter/package.scala +++ b/agni/core/app/foxcomm/agni/interpreter/package.scala @@ -1,10 +1,5 @@ package foxcomm.agni -import cats.~> -import monix.eval.{Coeval, Task} - package object interpreter { - implicit val coevalToTask: Coeval ~> Task = new (Coeval ~> Task) { - def apply[A](fa: Coeval[A]): Task[A] = fa.task - } + type Interpreter[A, B] = A ⇒ B } diff --git a/agni/core/app/foxcomm/agni/interpreter/query.scala b/agni/core/app/foxcomm/agni/interpreter/query.scala deleted file mode 100644 index 0ffe5622c8..0000000000 --- a/agni/core/app/foxcomm/agni/interpreter/query.scala +++ /dev/null @@ -1,81 +0,0 @@ -package foxcomm.agni.interpreter - -import scala.language.higherKinds -import cats.data.NonEmptyList -import cats.implicits._ -import cats.{~>, Monad} -import foxcomm.agni._ -import foxcomm.agni.dsl.query._ -import monix.eval.Coeval -import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} - -sealed abstract class QueryInterpreter[F[_]: Monad, V] extends ((V, NonEmptyList[QueryFunction]) ⇒ F[V]) { - final def eval(v: V, qf: QueryFunction): F[V] = qf match { - case qf: QueryFunction.matches ⇒ matchesF(v, qf) - case qf: QueryFunction.range ⇒ rangeF(v, qf) - case qf: QueryFunction.eq ⇒ eqF(v, qf) - case qf: QueryFunction.neq ⇒ neqF(v, qf) - } - - final def apply(v: V, qfs: NonEmptyList[QueryFunction]): F[V] = qfs.foldM(v)(eval) - - def matchesF(v: V, qf: QueryFunction.matches): F[V] - - def rangeF(v: V, qf: QueryFunction.range): F[V] - - def eqF(v: V, qf: QueryFunction.eq): F[V] - - def neqF(v: V, qf: QueryFunction.neq): F[V] -} - -trait LowPriorityQueryInterpreters { - @inline implicit def defaultQueryInterpreter: QueryInterpreter[Coeval, BoolQueryBuilder] = - DefaultQueryInterpreter -} - -object QueryInterpreter extends LowPriorityQueryInterpreters { - @inline implicit def apply[F[_], V](implicit qi: QueryInterpreter[F, V]): QueryInterpreter[F, V] = qi - - implicit class RichQueryInterpreter[F1[_], V](private val qi: QueryInterpreter[F1, V]) extends AnyVal { - def mapTo[F2[_]: Monad](implicit nat: F1 ~> F2): QueryInterpreter[F2, V] = new QueryInterpreter[F2, V] { - def matchesF(v: V, qf: QueryFunction.matches): F2[V] = nat(qi.matchesF(v, qf)) - - def rangeF(v: V, qf: QueryFunction.range): F2[V] = nat(qi.rangeF(v, qf)) - - def eqF(v: V, qf: QueryFunction.eq): F2[V] = nat(qi.eqF(v, qf)) - - def neqF(v: V, qf: QueryFunction.neq): F2[V] = nat(qi.neqF(v, qf)) - } - } -} - -object DefaultQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { - def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = Coeval.evalOnce { - val fields = qf.in.toList - qf.value.toList.foldLeft(b)((b, v) ⇒ b.must(QueryBuilders.multiMatchQuery(v, fields: _*))) - } - - def rangeF(b: BoolQueryBuilder, qf: QueryFunction.range): Coeval[BoolQueryBuilder] = Coeval.evalOnce { - val builder = QueryBuilders.rangeQuery(qf.in.field) - val value = qf.value.unify - value.lower.foreach { - case (RangeFunction.Gt, v) ⇒ builder.gt(v) - case (RangeFunction.Gte, v) ⇒ builder.gte(v) - } - value.upper.foreach { - case (RangeFunction.Lt, v) ⇒ builder.lt(v) - case (RangeFunction.Lte, v) ⇒ builder.lte(v) - } - b.filter(builder) - } - - def eqF(b: BoolQueryBuilder, qf: QueryFunction.eq): Coeval[BoolQueryBuilder] = Coeval.evalOnce { - val values = qf.value.toList - qf.in.toList.foldLeft(b)((b, n) ⇒ b.filter(QueryBuilders.termsQuery(n, values: _*))) - } - - def neqF(b: BoolQueryBuilder, qf: QueryFunction.neq): Coeval[BoolQueryBuilder] = Coeval.evalOnce { - val values = qf.value.toList - qf.in.toList.foldLeft(b)((b, n) ⇒ b.mustNot(QueryBuilders.termsQuery(n, values: _*))) - } -} diff --git a/agni/core/app/foxcomm/agni/package.scala b/agni/core/app/foxcomm/agni/package.scala index f234a9653b..8cd5ed30c8 100644 --- a/agni/core/app/foxcomm/agni/package.scala +++ b/agni/core/app/foxcomm/agni/package.scala @@ -1,19 +1,13 @@ package foxcomm -import cats.Monad import io.circe.generic.extras.Configuration -import monix.cats._ -import monix.eval.{Coeval, Task} +import monix.eval.Task import org.elasticsearch.action.ActionListener import scala.concurrent.Promise package object agni { implicit val configuration: Configuration = - Configuration.default.withDiscriminator("type").withSnakeCaseKeys - - // FIXME: For now we cache converted monix to cats monad in order to save unnecessary allocation on every call. - // Should be ready to remove once monix 3.0.x will become stable. - implicit val coevalMonad: Monad[Coeval] = monixToCatsMonad[Coeval] + Configuration.default.withDefaults.withDiscriminator("type").withSnakeCaseKeys @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) def async[A, B](action: ActionListener[A] ⇒ Any): Task[A] = Task.deferFuture { From 7bff2c7ee5a58b3356e69a99c37eddaf22aa0f3e Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Wed, 21 Jun 2017 17:20:32 +0200 Subject: [PATCH 24/61] Move query interpreter entirely to dedicated package --- agni/core/app/foxcomm/agni/SearchService.scala | 7 +------ agni/core/app/foxcomm/agni/dsl/query.scala | 8 +++++--- .../app/foxcomm/agni/interpreter/es/package.scala | 11 ++++++++--- agni/core/app/foxcomm/agni/payload.scala | 4 ++-- agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala | 2 +- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/agni/core/app/foxcomm/agni/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala index cbc03c1442..6c957537c5 100644 --- a/agni/core/app/foxcomm/agni/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -10,7 +10,6 @@ import org.elasticsearch.client.Client import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.transport.InetSocketTransportAddress -import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.SearchHit @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) @@ -37,11 +36,7 @@ class SearchService private (client: Client, qi: ESQueryInterpreter) { case SearchPayload.es(query, _) ⇒ Coeval.eval(builder.setQuery(Printer.noSpaces.prettyByteBuffer(Json.fromJsonObject(query)).array())) case SearchPayload.fc(query, _) ⇒ - query.fold { - Coeval.eval(builder.setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()))) - } { q ⇒ - qi(q.query).map(builder.setQuery) - } + qi(query).map(builder.setQuery) } def setupBuilder: Task[SearchRequestBuilder] = (prepareBuilder flatMap evalQuery).task diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index b5ed461af7..174167fc08 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -198,12 +198,14 @@ object query { implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } - final case class FCQuery(query: NonEmptyList[QueryFunction]) + final case class FCQuery(query: Option[NonEmptyList[QueryFunction]]) object FCQuery { implicit val decodeFCQuery: Decoder[FCQuery] = Decoder - .decodeNonEmptyList[QueryFunction] - .or(Decoder[QueryFunction].map(NonEmptyList.of(_))) + .decodeOption( + Decoder + .decodeNonEmptyList[QueryFunction] + .or(Decoder[QueryFunction].map(NonEmptyList.of(_)))) .map(FCQuery(_)) } } diff --git a/agni/core/app/foxcomm/agni/interpreter/es/package.scala b/agni/core/app/foxcomm/agni/interpreter/es/package.scala index d97febf3c9..f9d3138ae0 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/package.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/package.scala @@ -2,12 +2,17 @@ package foxcomm.agni.interpreter import cats.data.NonEmptyList import cats.implicits._ -import foxcomm.agni.dsl.query.QueryFunction +import foxcomm.agni.dsl.query.{FCQuery, QueryFunction} import monix.eval.Coeval import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} package object es { - type ESQueryInterpreter = Interpreter[NonEmptyList[QueryFunction], Coeval[BoolQueryBuilder]] + type ESQueryInterpreter = Interpreter[FCQuery, Coeval[BoolQueryBuilder]] - lazy val default: ESQueryInterpreter = ESQueryInterpreter <<< (QueryBuilders.boolQuery() → _) + lazy val default: ESQueryInterpreter = { + val eval: Interpreter[NonEmptyList[QueryFunction], Coeval[BoolQueryBuilder]] = + ESQueryInterpreter <<< (QueryBuilders.boolQuery() → _) + + _.query.fold(Coeval.eval(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery())))(eval) + } } diff --git a/agni/core/app/foxcomm/agni/payload.scala b/agni/core/app/foxcomm/agni/payload.scala index 46ce769c7f..38d0389df6 100644 --- a/agni/core/app/foxcomm/agni/payload.scala +++ b/agni/core/app/foxcomm/agni/payload.scala @@ -8,6 +8,6 @@ sealed trait SearchPayload { def fields: Option[NonEmptyList[String]] } object SearchPayload { - final case class es(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchPayload - final case class fc(query: Option[FCQuery], fields: Option[NonEmptyList[String]]) extends SearchPayload + final case class es(query: JsonObject, fields: Option[NonEmptyList[String]]) extends SearchPayload + final case class fc(query: FCQuery, fields: Option[NonEmptyList[String]]) extends SearchPayload } diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index 8ae70dfada..cc5e44a754 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -17,7 +17,7 @@ class QueryDslSpec extends FlatSpec with Matchers { "DSL" should "parse multiple queries" in { val json = parse(Source.fromInputStream(getClass.getResourceAsStream("/happy_path.json")).mkString).right.value - val queries = json.as[FCQuery].right.value.query.toList + val queries = json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) assertQueryFunction[QueryFunction.eq](queries.head) { is ⇒ is.in.toList should === (List("slug")) is.value.toList should === (List("awesome", "whatever")) From dc0f33304f0c6eb3ce1e0038af1db5d8b55dca1e Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Wed, 21 Jun 2017 17:36:54 +0200 Subject: [PATCH 25/61] Prefix agni endpoints with 'api' --- agni/api/app/foxcomm/agni/api/Api.scala | 2 +- agni/project/Settings.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/agni/api/app/foxcomm/agni/api/Api.scala b/agni/api/app/foxcomm/agni/api/Api.scala index e71f4de080..74055e0552 100644 --- a/agni/api/app/foxcomm/agni/api/Api.scala +++ b/agni/api/app/foxcomm/agni/api/Api.scala @@ -15,7 +15,7 @@ import scala.concurrent.ExecutionContext object Api extends App { def endpoints(searchService: SearchService)(implicit ec: ExecutionContext) = post( - "search" :: string :: string :: param("size") + "api" :: "search" :: string :: string :: param("size") .as[Int] :: paramOption("from").as[Int] :: jsonBody[SearchPayload]) { (searchIndex: String, searchType: String, size: Int, from: Option[Int], searchQuery: SearchPayload) ⇒ searchService diff --git a/agni/project/Settings.scala b/agni/project/Settings.scala index c1f080fc2f..66f7a4cd9a 100644 --- a/agni/project/Settings.scala +++ b/agni/project/Settings.scala @@ -52,8 +52,7 @@ object Settings { .cmdRaw(s"java $$JAVA_OPTS -jar $artifactTargetPath 2>&1 | tee -a /logs/${appName.value}.log") }, imageNames in docker := Seq( - ImageName( - s"${sys.props("DOCKER_REPO")}/${appName.value}:${sys.props("DOCKER_TAG")}") + ImageName(s"${sys.props("DOCKER_REPO")}/${appName.value}:${sys.props("DOCKER_TAG")}") ) ) } From 1ee9df70070cbbdf7f51cad7d2db1df18c797299 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Thu, 22 Jun 2017 09:56:10 +0200 Subject: [PATCH 26/61] Add raw query function Allow for raw es dsl usage on the function level. Model interpreter as kleisli. Refactor code around. --- .../core/app/foxcomm/agni/SearchService.scala | 2 +- agni/core/app/foxcomm/agni/dsl/query.scala | 71 +++++++++---------- .../agni/interpreter/QueryInterpreter.scala | 16 +++-- .../interpreter/es/ESQueryInterpreter.scala | 57 ++++++++------- .../foxcomm/agni/interpreter/es/package.scala | 8 +-- .../foxcomm/agni/interpreter/package.scala | 4 +- agni/core/app/foxcomm/agni/package.scala | 5 ++ .../test/foxcomm/agni/dsl/QueryDslSpec.scala | 10 +-- agni/core/test/resources/happy_path.json | 2 +- 9 files changed, 94 insertions(+), 81 deletions(-) diff --git a/agni/core/app/foxcomm/agni/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala index 6c957537c5..6c9c3fb3e3 100644 --- a/agni/core/app/foxcomm/agni/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -34,7 +34,7 @@ class SearchService private (client: Client, qi: ESQueryInterpreter) { def evalQuery(builder: SearchRequestBuilder): Coeval[SearchRequestBuilder] = searchQuery match { case SearchPayload.es(query, _) ⇒ - Coeval.eval(builder.setQuery(Printer.noSpaces.prettyByteBuffer(Json.fromJsonObject(query)).array())) + Coeval.eval(builder.setQuery(Json.fromJsonObject(query).dump)) case SearchPayload.fc(query, _) ⇒ qi(query).map(builder.setQuery) } diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 174167fc08..8e3590e8ed 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -1,16 +1,13 @@ package foxcomm.agni.dsl import cats.data.NonEmptyList -import cats.syntax.either._ +import cats.implicits._ +import io.circe._ import io.circe.generic.extras.semiauto._ -import io.circe.{Decoder, JsonNumber, KeyDecoder} import shapeless._ import shapeless.ops.coproduct.Folder -@SuppressWarnings( - Array("org.wartremover.warts.AsInstanceOf", - "org.wartremover.warts.ExplicitImplicitTypes", - "org.wartremover.warts.DefaultArguments")) +@SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf")) object query { sealed trait QueryField { def toList: List[String] @@ -20,12 +17,7 @@ object query { def toList: List[String] = List(field) } object Single { - implicit val decodeSingle: Decoder[Single] = Decoder.decodeString - .emap { - case s if s.startsWith("$") ⇒ Either.left(s"Defined unknown special query field $s") - case s ⇒ Either.right(s) - } - .map(Single(_)) + implicit val decodeSingle: Decoder[Single] = Decoder.decodeString.map(Single(_)) } final case class Multiple(fields: NonEmptyList[String]) extends QueryField { @@ -37,8 +29,8 @@ object query { } implicit val decodeQueryField: Decoder[QueryField] = - Decoder[Single].map(s ⇒ s: QueryField) or - Decoder[Multiple].map(m ⇒ m: QueryField) + Decoder[Single].map(identity[QueryField]) or + Decoder[Multiple].map(identity[QueryField]) } sealed trait QueryContext @@ -164,48 +156,55 @@ object query { sealed trait QueryFunction object QueryFunction { sealed trait WithField { this: QueryFunction ⇒ - def in: QueryField + def field: QueryField } sealed trait WithContext { this: QueryFunction ⇒ - def context: QueryContext + def ctx: QueryContext + } + sealed trait TermLevel extends WithContext { this: QueryFunction ⇒ + def context: Option[QueryContext] + def in: QueryField + + final def ctx: QueryContext = context.getOrElse(QueryContext.filter) + final def field: QueryField = in } + sealed trait FullText extends WithContext with WithField { this: QueryFunction ⇒ + def context: Option[QueryContext] + def in: Option[QueryField] - final case class matches private (in: QueryField, + final def ctx: QueryContext = context.getOrElse(QueryContext.must) + final def field: QueryField = in.getOrElse(QueryField.Single("_all")) + } + + final case class matches private (in: Option[QueryField], value: QueryValue[String], - context: QueryContext = QueryContext.must) + context: Option[QueryContext]) extends QueryFunction - with WithContext - with WithField - final case class range private (in: QueryField.Single, - value: RangeValue, - context: QueryContext = QueryContext.filter) + with FullText + final case class equals private (in: QueryField, value: CompoundValue, context: Option[QueryContext]) extends QueryFunction - with WithContext - with WithField - final case class eq private (in: QueryField, - value: CompoundValue, - context: QueryContext = QueryContext.filter) + with TermLevel + final case class range private (in: QueryField.Single, value: RangeValue, context: Option[QueryContext]) extends QueryFunction - with WithContext - with WithField - final case class neq private (in: QueryField, - value: CompoundValue, - context: QueryContext = QueryContext.not) + with TermLevel + final case class raw private (value: JsonObject, context: QueryContext) extends QueryFunction - with WithContext - with WithField + with WithContext { + def ctx: QueryContext = context + } implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } final case class FCQuery(query: Option[NonEmptyList[QueryFunction]]) object FCQuery { - implicit val decodeFCQuery: Decoder[FCQuery] = + implicit val decodeFCQuery: Decoder[FCQuery] = { Decoder .decodeOption( Decoder .decodeNonEmptyList[QueryFunction] .or(Decoder[QueryFunction].map(NonEmptyList.of(_)))) .map(FCQuery(_)) + } } } diff --git a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala index a82bec9e73..c9eb32c418 100644 --- a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala @@ -2,27 +2,29 @@ package foxcomm.agni.interpreter import scala.language.higherKinds import cats.Monad -import cats.data.NonEmptyList +import cats.data.{Kleisli, NonEmptyList} import cats.implicits._ import foxcomm.agni.dsl.query._ -abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[(V, NonEmptyList[QueryFunction]), F[V]] { +abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEmptyList[QueryFunction]), V] { + final def kleisli: Kleisli[F, (V, NonEmptyList[QueryFunction]), V] = Kleisli(this) + final def eval(v: V, qf: QueryFunction): F[V] = qf match { case qf: QueryFunction.matches ⇒ matchesF(v, qf) + case qf: QueryFunction.equals ⇒ equalsF(v, qf) case qf: QueryFunction.range ⇒ rangeF(v, qf) - case qf: QueryFunction.eq ⇒ eqF(v, qf) - case qf: QueryFunction.neq ⇒ neqF(v, qf) + case qf: QueryFunction.raw ⇒ rawF(v, qf) } final def apply(v: (V, NonEmptyList[QueryFunction])): F[V] = v._2.foldM(v._1)(eval) def matchesF(v: V, qf: QueryFunction.matches): F[V] - def rangeF(v: V, qf: QueryFunction.range): F[V] + def equalsF(v: V, qf: QueryFunction.equals): F[V] - def eqF(v: V, qf: QueryFunction.eq): F[V] + def rangeF(v: V, qf: QueryFunction.range): F[V] - def neqF(v: V, qf: QueryFunction.neq): F[V] + def rawF(v: V, qf: QueryFunction.raw): F[V] } object QueryInterpreter { diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala index e17af26c15..b7c6708267 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -1,39 +1,54 @@ package foxcomm.agni.interpreter.es +import foxcomm.agni._ import foxcomm.agni.dsl.query._ import foxcomm.agni.interpreter.QueryInterpreter +import io.circe.JsonObject import monix.cats._ import monix.eval.Coeval +import org.elasticsearch.common.xcontent.{ToXContent, XContentBuilder} import org.elasticsearch.index.query._ -@SuppressWarnings(Array("org.wartremover.warts.Overloading")) +@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { implicit class RichBoolQueryBuilder(val b: BoolQueryBuilder) extends AnyVal { - def inContext(qf: QueryFunction.WithContext)(qb: ⇒ QueryBuilder): BoolQueryBuilder = qf.context match { + def inContext(qf: QueryFunction.WithContext)(qb: ⇒ QueryBuilder): BoolQueryBuilder = qf.ctx match { case QueryContext.filter ⇒ b.filter(qb) case QueryContext.must ⇒ b.must(qb) case QueryContext.should ⇒ b.should(qb) case QueryContext.not ⇒ b.mustNot(qb) } + } - def foreachField(qf: QueryFunction.WithField)( - f: (BoolQueryBuilder, String) ⇒ BoolQueryBuilder): BoolQueryBuilder = { - qf.in.toList.foreach(f(b, _)) - b - } - - def foreachField(qf: QueryFunction.WithContext with QueryFunction.WithField)( - f: String ⇒ QueryBuilder): BoolQueryBuilder = { - qf.in.toList.foreach(n ⇒ b.inContext(qf)(f(n))) - b + final case class RawQueryBuilder(content: JsonObject) extends QueryBuilder { + def doXContent(builder: XContentBuilder, params: ToXContent.Params): Unit = { + builder.startObject() + content.toMap.foreach { + case (n, v) ⇒ + builder.rawField(n, v.dump) + } + builder.endObject() } } def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = Coeval.eval { - val fields = qf.in.toList qf.value.toList.foldLeft(b)((b, v) ⇒ b.inContext(qf) { - QueryBuilders.multiMatchQuery(v, fields: _*) + qf.field match { + case QueryField.Single(n) ⇒ QueryBuilders.matchQuery(n, v) + case QueryField.Multiple(ns) ⇒ QueryBuilders.multiMatchQuery(v, ns.toList: _*) + } + }) + } + + def equalsF(b: BoolQueryBuilder, qf: QueryFunction.equals): Coeval[BoolQueryBuilder] = Coeval.eval { + val vs = qf.value.toList + qf.field.toList.foldLeft(b)((b, n) ⇒ + b.inContext(qf) { + vs match { + case v :: Nil ⇒ QueryBuilders.termQuery(n, v) + case _ ⇒ QueryBuilders.termsQuery(n, vs: _*) + } }) } @@ -53,17 +68,7 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { } } - def eqF(b: BoolQueryBuilder, qf: QueryFunction.eq): Coeval[BoolQueryBuilder] = Coeval.eval { - val values = qf.value.toList - b.foreachField(qf) { n ⇒ - QueryBuilders.termsQuery(n, values: _*) - } - } - - def neqF(b: BoolQueryBuilder, qf: QueryFunction.neq): Coeval[BoolQueryBuilder] = Coeval.eval { - val values = qf.value.toList - b.foreachField(qf) { n ⇒ - QueryBuilders.termsQuery(n, values: _*) - } + def rawF(b: BoolQueryBuilder, qf: QueryFunction.raw): Coeval[BoolQueryBuilder] = Coeval.eval { + b.inContext(qf)(RawQueryBuilder(qf.value)) } } diff --git a/agni/core/app/foxcomm/agni/interpreter/es/package.scala b/agni/core/app/foxcomm/agni/interpreter/es/package.scala index f9d3138ae0..062e06cb8a 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/package.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/package.scala @@ -1,18 +1,18 @@ package foxcomm.agni.interpreter -import cats.data.NonEmptyList +import cats.data._ import cats.implicits._ import foxcomm.agni.dsl.query.{FCQuery, QueryFunction} import monix.eval.Coeval import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} package object es { - type ESQueryInterpreter = Interpreter[FCQuery, Coeval[BoolQueryBuilder]] + type ESQueryInterpreter = Kleisli[Coeval, FCQuery, BoolQueryBuilder] lazy val default: ESQueryInterpreter = { - val eval: Interpreter[NonEmptyList[QueryFunction], Coeval[BoolQueryBuilder]] = + val eval: Interpreter[Coeval, NonEmptyList[QueryFunction], BoolQueryBuilder] = ESQueryInterpreter <<< (QueryBuilders.boolQuery() → _) - _.query.fold(Coeval.eval(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery())))(eval) + Kleisli(_.query.fold(Coeval.eval(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery())))(eval)) } } diff --git a/agni/core/app/foxcomm/agni/interpreter/package.scala b/agni/core/app/foxcomm/agni/interpreter/package.scala index aae22a9f79..8acaa63617 100644 --- a/agni/core/app/foxcomm/agni/interpreter/package.scala +++ b/agni/core/app/foxcomm/agni/interpreter/package.scala @@ -1,5 +1,7 @@ package foxcomm.agni +import scala.language.higherKinds + package object interpreter { - type Interpreter[A, B] = A ⇒ B + type Interpreter[F[_], A, B] = A ⇒ F[B] } diff --git a/agni/core/app/foxcomm/agni/package.scala b/agni/core/app/foxcomm/agni/package.scala index 8cd5ed30c8..c28b5f629b 100644 --- a/agni/core/app/foxcomm/agni/package.scala +++ b/agni/core/app/foxcomm/agni/package.scala @@ -1,6 +1,7 @@ package foxcomm import io.circe.generic.extras.Configuration +import io.circe.{Json, Printer} import monix.eval.Task import org.elasticsearch.action.ActionListener import scala.concurrent.Promise @@ -19,4 +20,8 @@ package object agni { }) p.future } + + implicit class RichJson(val j: Json) extends AnyVal { + def dump: Array[Byte] = Printer.noSpaces.prettyByteBuffer(j).array() + } } diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index cc5e44a754..ef7daf2afd 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -18,16 +18,16 @@ class QueryDslSpec extends FlatSpec with Matchers { val json = parse(Source.fromInputStream(getClass.getResourceAsStream("/happy_path.json")).mkString).right.value val queries = json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) - assertQueryFunction[QueryFunction.eq](queries.head) { is ⇒ - is.in.toList should === (List("slug")) - is.value.toList should === (List("awesome", "whatever")) + assertQueryFunction[QueryFunction.equals](queries.head) { equals ⇒ + equals.field.toList should === (List("slug")) + equals.value.toList should === (List("awesome", "whatever")) } assertQueryFunction[QueryFunction.matches](queries(1)) { matches ⇒ - matches.in.toList should === (List("title", "description")) + matches.field.toList should === (List("title", "description")) matches.value.toList should === (List("food", "drink")) } assertQueryFunction[QueryFunction.range](queries(2)) { range ⇒ - range.in.toList should === (List("price")) + range.field.toList should === (List("price")) range.value.unify.toMap.mapValues(_.toString) should === ( Map( RangeFunction.Lt → "5000", diff --git a/agni/core/test/resources/happy_path.json b/agni/core/test/resources/happy_path.json index 630a71e829..54f424cb8a 100644 --- a/agni/core/test/resources/happy_path.json +++ b/agni/core/test/resources/happy_path.json @@ -1,6 +1,6 @@ [ { - "type": "eq", + "type": "equals", "in": "slug", "value": [ "awesome", "whatever" ] }, From 60b03ca96a72274fdbca5f88c7a5ab51944b0b0c Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Thu, 22 Jun 2017 12:15:52 +0200 Subject: [PATCH 27/61] Add exists function --- agni/core/app/foxcomm/agni/dsl/query.scala | 5 +++-- .../app/foxcomm/agni/interpreter/QueryInterpreter.scala | 3 +++ .../foxcomm/agni/interpreter/es/ESQueryInterpreter.scala | 9 ++++++++- agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala | 8 ++++++-- agni/core/test/resources/happy_path.json | 5 +++++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 8e3590e8ed..4b558183ff 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -163,10 +163,8 @@ object query { } sealed trait TermLevel extends WithContext { this: QueryFunction ⇒ def context: Option[QueryContext] - def in: QueryField final def ctx: QueryContext = context.getOrElse(QueryContext.filter) - final def field: QueryField = in } sealed trait FullText extends WithContext with WithField { this: QueryFunction ⇒ def context: Option[QueryContext] @@ -184,6 +182,9 @@ object query { final case class equals private (in: QueryField, value: CompoundValue, context: Option[QueryContext]) extends QueryFunction with TermLevel + final case class exists private (value: QueryField, context: Option[QueryContext]) + extends QueryFunction + with TermLevel final case class range private (in: QueryField.Single, value: RangeValue, context: Option[QueryContext]) extends QueryFunction with TermLevel diff --git a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala index c9eb32c418..23c60b7041 100644 --- a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala @@ -12,6 +12,7 @@ abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEm final def eval(v: V, qf: QueryFunction): F[V] = qf match { case qf: QueryFunction.matches ⇒ matchesF(v, qf) case qf: QueryFunction.equals ⇒ equalsF(v, qf) + case qf: QueryFunction.exists ⇒ existsF(v, qf) case qf: QueryFunction.range ⇒ rangeF(v, qf) case qf: QueryFunction.raw ⇒ rawF(v, qf) } @@ -22,6 +23,8 @@ abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEm def equalsF(v: V, qf: QueryFunction.equals): F[V] + def existsF(v: V, qf: QueryFunction.exists): F[V] + def rangeF(v: V, qf: QueryFunction.range): F[V] def rawF(v: V, qf: QueryFunction.raw): F[V] diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala index b7c6708267..ee86d6ad3d 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -43,7 +43,7 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { def equalsF(b: BoolQueryBuilder, qf: QueryFunction.equals): Coeval[BoolQueryBuilder] = Coeval.eval { val vs = qf.value.toList - qf.field.toList.foldLeft(b)((b, n) ⇒ + qf.in.toList.foldLeft(b)((b, n) ⇒ b.inContext(qf) { vs match { case v :: Nil ⇒ QueryBuilders.termQuery(n, v) @@ -52,6 +52,13 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { }) } + def existsF(b: BoolQueryBuilder, qf: QueryFunction.exists): Coeval[BoolQueryBuilder] = Coeval.eval { + qf.value.toList.foldLeft(b)((b, n) ⇒ + b.inContext(qf) { + QueryBuilders.existsQuery(n) + }) + } + def rangeF(b: BoolQueryBuilder, qf: QueryFunction.range): Coeval[BoolQueryBuilder] = Coeval.eval { b.inContext(qf) { val builder = QueryBuilders.rangeQuery(qf.in.field) diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index ef7daf2afd..19201401d6 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -19,7 +19,7 @@ class QueryDslSpec extends FlatSpec with Matchers { parse(Source.fromInputStream(getClass.getResourceAsStream("/happy_path.json")).mkString).right.value val queries = json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) assertQueryFunction[QueryFunction.equals](queries.head) { equals ⇒ - equals.field.toList should === (List("slug")) + equals.in.toList should === (List("slug")) equals.value.toList should === (List("awesome", "whatever")) } assertQueryFunction[QueryFunction.matches](queries(1)) { matches ⇒ @@ -27,12 +27,16 @@ class QueryDslSpec extends FlatSpec with Matchers { matches.value.toList should === (List("food", "drink")) } assertQueryFunction[QueryFunction.range](queries(2)) { range ⇒ - range.field.toList should === (List("price")) + range.in.toList should === (List("price")) range.value.unify.toMap.mapValues(_.toString) should === ( Map( RangeFunction.Lt → "5000", RangeFunction.Gte → "1000" )) } + assertQueryFunction[QueryFunction.exists](queries(3)) { exists ⇒ + exists.value.toList should === (List("archivedAt")) + exists.ctx should === (QueryContext.not) + } } } diff --git a/agni/core/test/resources/happy_path.json b/agni/core/test/resources/happy_path.json index 54f424cb8a..82777f2d80 100644 --- a/agni/core/test/resources/happy_path.json +++ b/agni/core/test/resources/happy_path.json @@ -16,5 +16,10 @@ "<": 5000, "gte": 1000 } + }, + { + "type": "exists", + "context": "not", + "value": "archivedAt" } ] From 27e23e114c6cbd41ea5676445fa113c1aafbb14f Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Thu, 22 Jun 2017 12:23:16 +0200 Subject: [PATCH 28/61] Check formatting on `make build` --- agni/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/agni/Makefile b/agni/Makefile index c4eddebf3d..5eb977c213 100644 --- a/agni/Makefile +++ b/agni/Makefile @@ -6,11 +6,14 @@ DOCKER_IMAGE ?= agni DOCKER_TAG ?= master SBT_CMD = sbt -DDOCKER_REPO=$(DOCKER_REPO) -DDOCKER_TAG=${DOCKER_TAG} +autoformat-check: + ../utils/scalafmt/scalafmt.sh --test + clean: $(call header, Cleaning) ${SBT_CMD} '; clean' -build: +build: autoformat-check $(call header, Building) ${SBT_CMD} '; compile; test:compile' From 538fa976e099165e4d684ce147270bcddb2b8876 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Thu, 22 Jun 2017 18:27:43 +0200 Subject: [PATCH 29/61] Add nested query function --- agni/core/app/foxcomm/agni/dsl/query.scala | 7 +++ .../agni/interpreter/QueryInterpreter.scala | 3 + .../interpreter/es/ESQueryInterpreter.scala | 60 +++++++++++-------- .../test/foxcomm/agni/dsl/QueryDslSpec.scala | 8 ++- 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 4b558183ff..7fc2ab8782 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -193,6 +193,13 @@ object query { with WithContext { def ctx: QueryContext = context } + final case class nested private (in: QueryField.Single, + context: QueryContext, + value: QueryValue[QueryFunction]) + extends QueryFunction + with WithContext { + def ctx: QueryContext = context + } implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } diff --git a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala index 23c60b7041..c0d0a3efba 100644 --- a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala @@ -15,6 +15,7 @@ abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEm case qf: QueryFunction.exists ⇒ existsF(v, qf) case qf: QueryFunction.range ⇒ rangeF(v, qf) case qf: QueryFunction.raw ⇒ rawF(v, qf) + case qf: QueryFunction.nested ⇒ nestedF(v, qf) } final def apply(v: (V, NonEmptyList[QueryFunction])): F[V] = v._2.foldM(v._1)(eval) @@ -28,6 +29,8 @@ abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEm def rangeF(v: V, qf: QueryFunction.range): F[V] def rawF(v: V, qf: QueryFunction.raw): F[V] + + def nestedF(v: V, qf: QueryFunction.nested): F[V] } object QueryInterpreter { diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala index ee86d6ad3d..2c507f48cf 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -1,5 +1,6 @@ package foxcomm.agni.interpreter.es +import cats.implicits._ import foxcomm.agni._ import foxcomm.agni.dsl.query._ import foxcomm.agni.interpreter.QueryInterpreter @@ -31,33 +32,37 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { } } - def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = Coeval.eval { - qf.value.toList.foldLeft(b)((b, v) ⇒ - b.inContext(qf) { - qf.field match { - case QueryField.Single(n) ⇒ QueryBuilders.matchQuery(n, v) - case QueryField.Multiple(ns) ⇒ QueryBuilders.multiMatchQuery(v, ns.toList: _*) - } - }) - } + def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = + Coeval.eval { + qf.value.toList.foldLeft(b)((b, v) ⇒ + b.inContext(qf) { + qf.field match { + case QueryField.Single(n) ⇒ QueryBuilders.matchQuery(n, v) + case QueryField.Multiple(ns) ⇒ + QueryBuilders.multiMatchQuery(v, ns.toList: _*) + } + }) + } - def equalsF(b: BoolQueryBuilder, qf: QueryFunction.equals): Coeval[BoolQueryBuilder] = Coeval.eval { - val vs = qf.value.toList - qf.in.toList.foldLeft(b)((b, n) ⇒ - b.inContext(qf) { - vs match { - case v :: Nil ⇒ QueryBuilders.termQuery(n, v) - case _ ⇒ QueryBuilders.termsQuery(n, vs: _*) - } - }) - } + def equalsF(b: BoolQueryBuilder, qf: QueryFunction.equals): Coeval[BoolQueryBuilder] = + Coeval.eval { + val vs = qf.value.toList.toList + qf.in.toList.foldLeft(b)((b, n) ⇒ + b.inContext(qf) { + vs match { + case v :: Nil ⇒ QueryBuilders.termQuery(n, v) + case _ ⇒ QueryBuilders.termsQuery(n, vs: _*) + } + }) + } - def existsF(b: BoolQueryBuilder, qf: QueryFunction.exists): Coeval[BoolQueryBuilder] = Coeval.eval { - qf.value.toList.foldLeft(b)((b, n) ⇒ - b.inContext(qf) { - QueryBuilders.existsQuery(n) - }) - } + def existsF(b: BoolQueryBuilder, qf: QueryFunction.exists): Coeval[BoolQueryBuilder] = + Coeval.eval { + qf.value.toList.foldLeft(b)((b, n) ⇒ + b.inContext(qf) { + QueryBuilders.existsQuery(n) + }) + } def rangeF(b: BoolQueryBuilder, qf: QueryFunction.range): Coeval[BoolQueryBuilder] = Coeval.eval { b.inContext(qf) { @@ -78,4 +83,9 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { def rawF(b: BoolQueryBuilder, qf: QueryFunction.raw): Coeval[BoolQueryBuilder] = Coeval.eval { b.inContext(qf)(RawQueryBuilder(qf.value)) } + + def nestedF(b: BoolQueryBuilder, qf: QueryFunction.nested): Coeval[BoolQueryBuilder] = + qf.value.toList + .foldM(QueryBuilders.boolQuery())(eval) + .map(b.inContext(qf)(_)) } diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index 19201401d6..efef16b589 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -16,8 +16,12 @@ class QueryDslSpec extends FlatSpec with Matchers { "DSL" should "parse multiple queries" in { val json = - parse(Source.fromInputStream(getClass.getResourceAsStream("/happy_path.json")).mkString).right.value - val queries = json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) + parse( + Source + .fromInputStream(getClass.getResourceAsStream("/happy_path.json")) + .mkString).right.value + val queries = + json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) assertQueryFunction[QueryFunction.equals](queries.head) { equals ⇒ equals.in.toList should === (List("slug")) equals.value.toList should === (List("awesome", "whatever")) From 5d0f86d1d6b11f134187b8b59e9141d99b479511 Mon Sep 17 00:00:00 2001 From: Artem Date: Wed, 14 Jun 2017 17:16:43 +0300 Subject: [PATCH 30/61] Add configs for the search service --- tabernacle/ansible/group_vars/all | 14 ++++- .../templates/core-backend/agni.json.j2 | 56 +++++++++++++++++++ .../templates/highlander.json.j2 | 1 + 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 diff --git a/tabernacle/ansible/group_vars/all b/tabernacle/ansible/group_vars/all index cefc5fb78d..06cf1ee780 100644 --- a/tabernacle/ansible/group_vars/all +++ b/tabernacle/ansible/group_vars/all @@ -51,7 +51,8 @@ gce_zone: "us-central1-a" # Default docker tags for containers docker_tags: ashes: "{{ lookup('env', 'DOCKER_TAG_ASHES') | default('master', true) }}" - firebrand: "{{ lookup('env', 'DOCKER_TAG_FIREBRAND') | default('master', true) }}" + agni: "{{ lookup('env', 'DOCKER_TAG_AGNI') | default('master', true) }}" + firebrand: "{{ lookup('env', 'DOCKER_TAG_FIREBRAND') | default('master', true) }}" peacock: "{{ lookup('env', 'DOCKER_TAG_PEACOCK') | default('master', true) }}" phoenix: "{{ lookup('env', 'DOCKER_TAG_PHOENIX') | default('master', true) }}" phoenix_seeder: "{{ lookup('env', 'DOCKER_TAG_PHOENIX_SEEDER') | default('master', true) }}" @@ -89,7 +90,8 @@ docker_tags: # Configurable Marathon re-deploys marathon_restart: ashes: "{{ lookup('env', 'MARATHON_ASHES') | default(true, true) | bool }}" - firebrand: "{{ lookup('env', 'MARATHON_FIREBRAND') | default(true, true) | bool }}" + agni: "{{ lookup('env', 'MARATHON_AGNI') | default(true, true) | bool }}" + firebrand: "{{ lookup('env', 'MARATHON_FIREBRAND') | default(true, true) | bool }}" peacock: "{{ lookup('env', 'MARATHON_PEACOCK') | default(true, true) | bool }}" phoenix: "{{ lookup('env', 'MARATHON_PHOENIX') | default(true, true) | bool }}" greenriver: "{{ lookup('env', 'MARATHON_GREENRIVER') | default(true, true) | bool }}" @@ -141,6 +143,10 @@ phoenix_host: "phoenix.{{consul_suffix}}" phoenix_port: 9090 phoenix_server: "{{phoenix_host}}:{{phoenix_port}}" +agni_host: "agni.{{consul_suffix}}" +agni_port: 9000 +agni_server: "{{agni_host}}:{{agni_port}}" + isaac_host: "isaac.{{consul_suffix}}" isaac_port: 9190 isaac_server: "{{isaac_host}}:{{isaac_port}}" @@ -266,6 +272,10 @@ phoenix_api_password: "api$pass7!" phoenix_tax_rule_region: 4129 phoenix_tax_rule_rate: 7.5 +# Agni +agni_src: ../../search-service +agni_dir: /search-service + # Green River greenriver_env: localhost greenriver_restart: "false" diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 new file mode 100644 index 0000000000..82ab23fedd --- /dev/null +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -0,0 +1,56 @@ +{ + "id": "agni", + "cmd": null, + "cpus": 0.5, + "mem": 256, + "disk": 0, + "instances": 1, + "constraints": [], + "labels": { + "MARATHON_SINGLE_INSTANCE_APP": "true", + "LANG": "scala", + "consul": "agni", + "overrideTaskName": "agni", + "TAG": "{{docker_tags.agni}}" + }, + "upgradeStrategy": { + "minimumHealthCapacity": 0, + "maximumOverCapacity": 0 + }, + "env": { + "PORT": "{{agni_port}}", + "SEARCH_SERVER": "" + }, + "ports": [{{agni_port}}], + "healthChecks": [{ + "path": "/ping", + "protocol": "HTTP", + "gracePeriodSeconds": 300, + "intervalSeconds": 30, + "timeoutSeconds": 20, + "maxConsecutiveFailures": 3, + "ignoreHttp1xx": false, + "port": {{agni_port}} + }], + "container": { + "type": "DOCKER", + "volumes": [{ + "containerPath": "/keys", + "hostPath": "{{public_keys_dest_dir}}", + "mode": "RO" + }, + { + "containerPath": "{{docker_logs_dir}}", + "hostPath": "{{docker_logs_host_dir}}", + "mode": "RW" + } + ], + "docker": { + "image": "{{docker_registry}}:5000/agni:{{docker_tags.agni}}", + "network": "HOST", + "privileged": false, + "parameters": [], + "forcePullImage": true + } + } +} diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 index 644cad40bf..e45765cbaa 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 @@ -9,6 +9,7 @@ {% include "core-backend/isaac.json.j2" %}, {% include "core-backend/solomon.json.j2" %}, {% include "core-backend/middlewarehouse.json.j2" %} + {% include "core-backend/agni.json.j2" %} ] }, { From d4b912b0b629fe0a68567a3112f4a84602e7607c Mon Sep 17 00:00:00 2001 From: Artem Date: Wed, 14 Jun 2017 17:33:02 +0300 Subject: [PATCH 31/61] Update configs --- tabernacle/ansible/group_vars/all | 8 ++++---- .../marathon_groups/templates/core-backend/agni.json.j2 | 7 +------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tabernacle/ansible/group_vars/all b/tabernacle/ansible/group_vars/all index 06cf1ee780..7731130e40 100644 --- a/tabernacle/ansible/group_vars/all +++ b/tabernacle/ansible/group_vars/all @@ -51,8 +51,8 @@ gce_zone: "us-central1-a" # Default docker tags for containers docker_tags: ashes: "{{ lookup('env', 'DOCKER_TAG_ASHES') | default('master', true) }}" - agni: "{{ lookup('env', 'DOCKER_TAG_AGNI') | default('master', true) }}" - firebrand: "{{ lookup('env', 'DOCKER_TAG_FIREBRAND') | default('master', true) }}" + agni: "{{ lookup('env', 'DOCKER_TAG_AGNI') | default('master', true) }}" + firebrand: "{{ lookup('env', 'DOCKER_TAG_FIREBRAND') | default('master', true) }}" peacock: "{{ lookup('env', 'DOCKER_TAG_PEACOCK') | default('master', true) }}" phoenix: "{{ lookup('env', 'DOCKER_TAG_PHOENIX') | default('master', true) }}" phoenix_seeder: "{{ lookup('env', 'DOCKER_TAG_PHOENIX_SEEDER') | default('master', true) }}" @@ -90,8 +90,8 @@ docker_tags: # Configurable Marathon re-deploys marathon_restart: ashes: "{{ lookup('env', 'MARATHON_ASHES') | default(true, true) | bool }}" - agni: "{{ lookup('env', 'MARATHON_AGNI') | default(true, true) | bool }}" - firebrand: "{{ lookup('env', 'MARATHON_FIREBRAND') | default(true, true) | bool }}" + agni: "{{ lookup('env', 'MARATHON_AGNI') | default(true, true) | bool }}" + firebrand: "{{ lookup('env', 'MARATHON_FIREBRAND') | default(true, true) | bool }}" peacock: "{{ lookup('env', 'MARATHON_PEACOCK') | default(true, true) | bool }}" phoenix: "{{ lookup('env', 'MARATHON_PHOENIX') | default(true, true) | bool }}" greenriver: "{{ lookup('env', 'MARATHON_GREENRIVER') | default(true, true) | bool }}" diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index 82ab23fedd..3d5e029c8a 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -19,7 +19,7 @@ }, "env": { "PORT": "{{agni_port}}", - "SEARCH_SERVER": "" + "SEARCH_SERVER": "elasticsearch://{{search_server}}", }, "ports": [{{agni_port}}], "healthChecks": [{ @@ -35,11 +35,6 @@ "container": { "type": "DOCKER", "volumes": [{ - "containerPath": "/keys", - "hostPath": "{{public_keys_dest_dir}}", - "mode": "RO" - }, - { "containerPath": "{{docker_logs_dir}}", "hostPath": "{{docker_logs_host_dir}}", "mode": "RW" From efc7cca3e919b4c8102cab845e2540216fed69e9 Mon Sep 17 00:00:00 2001 From: Artem Date: Wed, 14 Jun 2017 17:40:55 +0300 Subject: [PATCH 32/61] Spacing --- .../roles/dev/marathon_groups/templates/highlander.json.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 index e45765cbaa..f1abf4bb96 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 @@ -9,7 +9,7 @@ {% include "core-backend/isaac.json.j2" %}, {% include "core-backend/solomon.json.j2" %}, {% include "core-backend/middlewarehouse.json.j2" %} - {% include "core-backend/agni.json.j2" %} + {% include "core-backend/agni.json.j2" %} ] }, { From 00902d8f248579578c6431eed92f77195fead9d4 Mon Sep 17 00:00:00 2001 From: Artem Date: Wed, 14 Jun 2017 18:15:24 +0300 Subject: [PATCH 33/61] Add hotfix for loading new rsyslog config --- .../roles/hotfix/rsyslog/tasks/main.yml | 5 ++ .../hotfix/rsyslog/templates/mesos.conf.j2 | 77 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml create mode 100644 tabernacle/ansible/roles/hotfix/rsyslog/templates/mesos.conf.j2 diff --git a/tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml b/tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml new file mode 100644 index 0000000000..333c23ee30 --- /dev/null +++ b/tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml @@ -0,0 +1,5 @@ +- name: Remove Current File Data Forwarding Config + file: path=/etc/rsyslog.d/51-mesos.conf state=absent + +- name: Install New File Data Forwarding Config + template: src=mesos.conf.j2 dest=/etc/rsyslog.d/51-mesos.conf owner=root group=root mode=0644 diff --git a/tabernacle/ansible/roles/hotfix/rsyslog/templates/mesos.conf.j2 b/tabernacle/ansible/roles/hotfix/rsyslog/templates/mesos.conf.j2 new file mode 100644 index 0000000000..7295eb724d --- /dev/null +++ b/tabernacle/ansible/roles/hotfix/rsyslog/templates/mesos.conf.j2 @@ -0,0 +1,77 @@ +module(load="imfile" PollingInterval="10") + +# application logs in docker containers +input(type="imfile" + File="/var/log/docker/phoenix.log" + Tag="phoenix" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/middlewarehouse.log" + Tag="middlewarehouse" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/ashes.log" + Tag="ashes" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/td-storefront.log" + Tag="storefront" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/tpg-storefront.log" + Tag="storefront" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/firebrand.log" + Tag="storefront" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/isaac.log" + Tag="isaac" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/solomon.log" + Tag="solomon" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/green-river.log" + Tag="green-river" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/messaging.log" + Tag="messaging" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/capture-consumer.log" + Tag="capture-consumer" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/gift-card-consumer.log" + Tag="gift-card-consumer" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/shipment-consumer.log" + Tag="shipment-consumer" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/stock-items-consumer.log" + Tag="stock-items-consumer" + Facility="local6") + +input(type="imfile" + File="/var/log/docker/agni.log" + Tag="agni" + Facility="local6") From 25e377e1ca1ffaea01a1a8ddf8f2c221edf6c073 Mon Sep 17 00:00:00 2001 From: Artem Date: Thu, 15 Jun 2017 20:02:42 +0300 Subject: [PATCH 34/61] More configs --- .../app/config_gen/templates/goldrush.cfg.j2 | 2 ++ .../roles/app/deploy_helper/vars/main.yml | 2 ++ .../balancer/templates/service_locations.j2 | 24 +++++++++++++++++++ .../roles/dev/balancer/templates/services.j2 | 6 +++++ .../roles/dev/marathon_restart/tasks/main.yml | 1 + .../roles/dev/marathon_restart/vars/main.yml | 1 + .../dev/nginx/templates/service_locations.j2 | 24 +++++++++++++++++++ .../roles/dev/nginx/templates/services.j2 | 5 ++++ .../roles/hotfix/rsyslog/tasks/main.yml | 3 +++ .../templates/pipeline.json.j2 | 1 + 10 files changed, 69 insertions(+) diff --git a/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 b/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 index 3ab8aab6d7..399552cde8 100644 --- a/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 +++ b/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 @@ -16,6 +16,7 @@ export WITH_APPLIANCE_SEEDING={{with_seeding_value | bool | lower}} # Core export DOCKER_TAG_ASHES:=master +export DOCKER_TAG_AGNI:=master export DOCKER_TAG_PEACOCK:=master export DOCKER_TAG_PHOENIX:=master export DOCKER_TAG_GREENRIVER:=master @@ -66,6 +67,7 @@ export DOCKER_TAG_DATA_IMPORT=master #################################################################### # Core +export MARATHON_AGNI:=false export MARATHON_ASHES:=false export MARATHON_PEACOCK:=false export MARATHON_PHOENIX:=false diff --git a/tabernacle/ansible/roles/app/deploy_helper/vars/main.yml b/tabernacle/ansible/roles/app/deploy_helper/vars/main.yml index db099ad5e8..57cfa3e426 100644 --- a/tabernacle/ansible/roles/app/deploy_helper/vars/main.yml +++ b/tabernacle/ansible/roles/app/deploy_helper/vars/main.yml @@ -7,6 +7,7 @@ input_aliases: # Supported canonical apps (otherwise, playbook fails fast) supported_apps: - ashes + - agni - peacock - phoenix - greenriver @@ -75,6 +76,7 @@ app_push_targets: # Override default Docker tags docker_tags: + agni: "{{tag_name}}" ashes: "{{tag_name}}" peacock: "{{tag_name}}" phoenix: "{{tag_name}}" diff --git a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 index c97a617d67..c2f63bd60a 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 @@ -144,6 +144,30 @@ location ~ /api/search/public/.*/\d+$ { break; } +# Proxy the internal search location and sanitizes output from ES for external use +location ~ /api/v1/advanced-search/.*/\d+$ { + auth_request /internal-auth; + default_type 'application/json'; + content_by_lua ' + + --forward to internal es + local json = request_internal_search("admin") --what the hell to put here?? + + if json.found == true then + --only show _source + json = json._source + + ngx.say(j.encode(json)) + else + ngx.status = ngx.HTTP_NOT_FOUND + ngx.header["Content-type"] = "application/json" + json = { found = false} + ngx.say(j.encode(json)) + end + '; + break; +} + # Proxy to middlewarehouse location /api/v1/inventory/ { auth_request /internal-auth; diff --git a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 index 17e3b5450a..b2395c31a6 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 @@ -58,6 +58,12 @@ upstream sol { << else >> server {{solomon_server}} fail_timeout=30s max_fails=10; << end >> } +# We use port 9000 for agni +upstream agni { + << range service "agni" >> server << .Address >>:9000 max_fails=10 fail_timeout=30s weight=1; + << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> +} + upstream hyperion { << range service "hyperion" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; << else >> server {{hyperion_server}} fail_timeout=30s max_fails=10; << end >> diff --git a/tabernacle/ansible/roles/dev/marathon_restart/tasks/main.yml b/tabernacle/ansible/roles/dev/marathon_restart/tasks/main.yml index 9801296f4e..54d56196e8 100644 --- a/tabernacle/ansible/roles/dev/marathon_restart/tasks/main.yml +++ b/tabernacle/ansible/roles/dev/marathon_restart/tasks/main.yml @@ -12,6 +12,7 @@ - { group: core-backend, app: middlewarehouse, id: middlewarehouse } - { group: core-backend, app: isaac, id: isaac } - { group: core-backend, app: solomon, id: solomon } + - { group: core-backend, app: agni, id: agni } - { group: core-frontend, app: ashes, id: ashes } - { group: core-frontend, app: peacock, id: peacock } - { group: core-frontend, app: perfect-gourmet, id: storefront_tpg } diff --git a/tabernacle/ansible/roles/dev/marathon_restart/vars/main.yml b/tabernacle/ansible/roles/dev/marathon_restart/vars/main.yml index f7b92bd65d..de49d82da9 100644 --- a/tabernacle/ansible/roles/dev/marathon_restart/vars/main.yml +++ b/tabernacle/ansible/roles/dev/marathon_restart/vars/main.yml @@ -3,6 +3,7 @@ marathon_deploy: isaac: "{{ lookup('env', 'MRT_ISAAC') | default(false, true) | bool }}" solomon: "{{ lookup('env', 'MRT_SOLOMON') | default(false, true) | bool }}" middlewarehouse: "{{ lookup('env', 'MRT_MIDDLEWAREHOUSE') | default(false, true) | bool }}" + agni: "{{ lookup('env', 'MRT_AGNI') | default(false, true) | bool }}" greenriver: "{{ lookup('env', 'MRT_GREENRIVER') | default(false, true) | bool }}" capture_consumer: "{{ lookup('env', 'MRT_CAPTURE_CONSUMER') | default(false, true) | bool }}" diff --git a/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 index 0cd1329aca..1ee070fa87 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 @@ -121,6 +121,30 @@ location ~ /api/search/public/.*/_count { break; } +# Proxy the internal search location and sanitizes output from ES for external use +location ~ /api/v1/advanced-search/.*/\d+$ { + auth_request /internal-auth; + default_type 'application/json'; + content_by_lua ' + + --forward to internal es + local json = request_internal_search("admin") --what the hell to put here?? + + if json.found == true then + --only show _source + json = json._source + + ngx.say(j.encode(json)) + else + ngx.status = ngx.HTTP_NOT_FOUND + ngx.header["Content-type"] = "application/json" + json = { found = false} + ngx.say(j.encode(json)) + end + '; + break; +} + # Proxy the internal search location and sanitizes output from ES for external use location ~ /api/search/public/.*/\d+$ { default_type 'application/json'; diff --git a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 index 17e3b5450a..4b37dbb5a0 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 @@ -66,6 +66,11 @@ upstream hyperion { upstream geronimo { << range service "geronimo" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; << else >> server {{geronimo_server}} fail_timeout=30s max_fails=10; << end >> + +# We use port 9000 for agni +upstream agni { + << range service "agni" >> server << .Address >>:9000 max_fails=10 fail_timeout=30s weight=1; + << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> } upstream ashes { diff --git a/tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml b/tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml index 333c23ee30..c6384bbcc2 100644 --- a/tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml +++ b/tabernacle/ansible/roles/hotfix/rsyslog/tasks/main.yml @@ -3,3 +3,6 @@ - name: Install New File Data Forwarding Config template: src=mesos.conf.j2 dest=/etc/rsyslog.d/51-mesos.conf owner=root group=root mode=0644 + +- name: Restart rsyslog + service: name=rsyslog state=restarted diff --git a/tabernacle/ansible/roles/ops/buildkite_pipeline/templates/pipeline.json.j2 b/tabernacle/ansible/roles/ops/buildkite_pipeline/templates/pipeline.json.j2 index f282abe264..ba8ade1349 100644 --- a/tabernacle/ansible/roles/ops/buildkite_pipeline/templates/pipeline.json.j2 +++ b/tabernacle/ansible/roles/ops/buildkite_pipeline/templates/pipeline.json.j2 @@ -24,6 +24,7 @@ "APPLIANCE_DNS_RECORD": "feature-branch-{{docker_tag_name}}", "DOCKER_TAG_PHOENIX": "{{docker_tag_name}}", "MARATHON_PHOENIX": "true", + "MARATHON_AGNI": "false", "MARATHON_ASHES": "false", "MARATHON_PEACOCK": "false", "MARATHON_GREENRIVER": "false", From f652db823ca1f6db8e7a9e9f2b874545018b07fe Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 19 Jun 2017 19:44:51 +0300 Subject: [PATCH 35/61] Add extra configs --- Makefile | 2 +- .../roles/dev/balancer/templates/service_locations.j2 | 8 +++++++- .../marathon_groups/templates/core-backend/agni.json.j2 | 2 +- .../dev/marathon_groups/templates/highlander.json.j2 | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6a5d871704..5f60274c0a 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ config: up: $(call header, Creating Appliance) - ansible-playbook --user=$(GOOGLE_SSH_USERNAME) --private-key=$(GOOGLE_SSH_KEY) --extra-vars '{"FIRST_RUN": true}' tabernacle/ansible/goldrush_appliance.yml + ansible-playbook -vvvv --user=$(GOOGLE_SSH_USERNAME) --private-key=$(GOOGLE_SSH_KEY) --extra-vars '{"FIRST_RUN": false}' tabernacle/ansible/goldrush_appliance.yml @cat goldrush.log destroy: diff --git a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 index c2f63bd60a..992fd0951a 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 @@ -145,9 +145,15 @@ location ~ /api/search/public/.*/\d+$ { } # Proxy the internal search location and sanitizes output from ES for external use -location ~ /api/v1/advanced-search/.*/\d+$ { +location ~ /api/advanced-search/.*/\d+$ { auth_request /internal-auth; default_type 'application/json'; + proxy_pass http://agni/search/; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; content_by_lua ' --forward to internal es diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index 3d5e029c8a..3121e8294f 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -19,7 +19,7 @@ }, "env": { "PORT": "{{agni_port}}", - "SEARCH_SERVER": "elasticsearch://{{search_server}}", + "SEARCH_SERVER": "elasticsearch://{{search_server}}" }, "ports": [{{agni_port}}], "healthChecks": [{ diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 index f1abf4bb96..cf28a5811d 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 @@ -8,7 +8,7 @@ {% include "core-backend/phoenix.json.j2" %}, {% include "core-backend/isaac.json.j2" %}, {% include "core-backend/solomon.json.j2" %}, - {% include "core-backend/middlewarehouse.json.j2" %} + {% include "core-backend/middlewarehouse.json.j2" %}, {% include "core-backend/agni.json.j2" %} ] }, From 634a7e1c483a040ec2add3c2dd9ef7fcf1ab7c8f Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 19 Jun 2017 19:52:04 +0300 Subject: [PATCH 36/61] Fix builder.py and vars in tabernacle --- tabernacle/ansible/group_vars/all | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tabernacle/ansible/group_vars/all b/tabernacle/ansible/group_vars/all index 7731130e40..950db704ea 100644 --- a/tabernacle/ansible/group_vars/all +++ b/tabernacle/ansible/group_vars/all @@ -273,8 +273,8 @@ phoenix_tax_rule_region: 4129 phoenix_tax_rule_rate: 7.5 # Agni -agni_src: ../../search-service -agni_dir: /search-service +agni_src: ../../agni +agni_dir: /agni # Green River greenriver_env: localhost From e7b3418d1eaddb5ef3ff80da8db325b55e9de6fb Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 19 Jun 2017 20:03:55 +0300 Subject: [PATCH 37/61] Change nginx proxy --- .../balancer/templates/service_locations.j2 | 23 +++------------- .../dev/nginx/templates/service_locations.j2 | 27 ++++++------------- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 index 992fd0951a..6c30d0e4af 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 @@ -144,33 +144,16 @@ location ~ /api/search/public/.*/\d+$ { break; } -# Proxy the internal search location and sanitizes output from ES for external use -location ~ /api/advanced-search/.*/\d+$ { +# Proxy to agni +location /api/advanced-search/.*/\d+$ { auth_request /internal-auth; default_type 'application/json'; - proxy_pass http://agni/search/; + proxy_pass http://agni/search/; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - content_by_lua ' - - --forward to internal es - local json = request_internal_search("admin") --what the hell to put here?? - - if json.found == true then - --only show _source - json = json._source - - ngx.say(j.encode(json)) - else - ngx.status = ngx.HTTP_NOT_FOUND - ngx.header["Content-type"] = "application/json" - json = { found = false} - ngx.say(j.encode(json)) - end - '; break; } diff --git a/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 index 1ee070fa87..4c41d6337a 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 @@ -121,27 +121,16 @@ location ~ /api/search/public/.*/_count { break; } -# Proxy the internal search location and sanitizes output from ES for external use -location ~ /api/v1/advanced-search/.*/\d+$ { +# Proxy to agni +location /api/advanced-search/.*/\d+$ { auth_request /internal-auth; default_type 'application/json'; - content_by_lua ' - - --forward to internal es - local json = request_internal_search("admin") --what the hell to put here?? - - if json.found == true then - --only show _source - json = json._source - - ngx.say(j.encode(json)) - else - ngx.status = ngx.HTTP_NOT_FOUND - ngx.header["Content-type"] = "application/json" - json = { found = false} - ngx.say(j.encode(json)) - end - '; + proxy_pass http://agni/search/; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; break; } From 1380abe6abb8fbcc30571ac1107b3fde97bd5d1e Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Fri, 23 Jun 2017 13:11:49 +0200 Subject: [PATCH 38/61] Support nested fields Allow to specify nested fields in query functions. Change folding over coproduct. Remove monad constraint from query interpreter. Refactor code around. --- agni/core/app/foxcomm/agni/dsl/query.scala | 134 ++++++++---------- .../agni/interpreter/QueryInterpreter.scala | 10 +- .../interpreter/es/ESQueryInterpreter.scala | 80 ++++++----- .../foxcomm/agni/interpreter/es/package.scala | 3 +- .../test/foxcomm/agni/dsl/QueryDslSpec.scala | 16 ++- agni/core/test/resources/happy_path.json | 2 +- 6 files changed, 121 insertions(+), 124 deletions(-) diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 7fc2ab8782..14b686f658 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -1,31 +1,68 @@ package foxcomm.agni.dsl -import cats.data.NonEmptyList +import cats.data.{NonEmptyList, NonEmptyVector} import cats.implicits._ import io.circe._ import io.circe.generic.extras.semiauto._ import shapeless._ -import shapeless.ops.coproduct.Folder -@SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf")) +@SuppressWarnings(Array("org.wartremover.warts.Equals")) object query { + type QueryValue[T] = T :+: NonEmptyList[T] :+: CNil + type CompoundValue = QueryValue[JsonNumber] :+: QueryValue[String] :+: CNil + type Field = String :+: NonEmptyVector[String] :+: CNil + type RangeValue = RangeBound[JsonNumber] :+: RangeBound[String] :+: CNil + + implicit class RichQueryValue[T](val qv: QueryValue[T]) extends AnyVal { + def toNEL: NonEmptyList[T] = qv.eliminate(NonEmptyList.of(_), _.eliminate(identity, _.impossible)) + + def toList: List[T] = toNEL.toList + } + + implicit class RichCompoundValue(val cv: CompoundValue) extends AnyVal { + def toNEL: NonEmptyList[AnyRef] = cv.eliminate(_.toNEL, _.eliminate(_.toNEL, _.impossible)) + + def toList: List[AnyRef] = toNEL.toList + } + + implicit def decodeQueryValue[T: Decoder]: Decoder[QueryValue[T]] = + Decoder[T].map(Coproduct[QueryValue[T]](_)) or Decoder + .decodeNonEmptyList[T] + .map(Coproduct[QueryValue[T]](_)) + + implicit val decodeCompoundValue: Decoder[CompoundValue] = + Decoder[QueryValue[JsonNumber]].map(Coproduct[CompoundValue](_)) or + Decoder[QueryValue[String]].map(Coproduct[CompoundValue](_)) + + implicit val decodeField: Decoder[Field] = Decoder.decodeString.map { s ⇒ + val xs = s.split("\\.") + if (xs.length > 1) Coproduct[Field](NonEmptyVector.of(xs.head, xs.tail: _*)) + else Coproduct[Field](s) + } + + implicit val decodeRange: Decoder[RangeValue] = + Decoder[RangeBound[JsonNumber]].map(Coproduct[RangeValue](_)) or + Decoder[RangeBound[String]].map(Coproduct[RangeValue](_)) + sealed trait QueryField { - def toList: List[String] + def toNEL: NonEmptyList[Field] + + def toList: List[Field] = toNEL.toList } object QueryField { - final case class Single(field: String) extends QueryField { - def toList: List[String] = List(field) + final case class Single(field: Field) extends QueryField { + def toNEL: NonEmptyList[Field] = NonEmptyList.of(field) } object Single { - implicit val decodeSingle: Decoder[Single] = Decoder.decodeString.map(Single(_)) + implicit val decodeSingle: Decoder[Single] = Decoder[Field].map(Single(_)) } - final case class Multiple(fields: NonEmptyList[String]) extends QueryField { - def toList: List[String] = fields.toList + final case class Multiple(fields: NonEmptyList[Field]) extends QueryField { + def toNEL: NonEmptyList[Field] = fields } object Multiple { implicit val decodeMultiple: Decoder[Multiple] = - Decoder.decodeNonEmptyList[String].map(Multiple(_)) + Decoder.decodeNonEmptyList[Field].map(Multiple(_)) } implicit val decodeQueryField: Decoder[QueryField] = @@ -74,9 +111,7 @@ object query { } final case class RangeBound[T](lower: Option[(RangeFunction.LowerBound, T)], - upper: Option[(RangeFunction.UpperBound, T)]) { - def toMap: Map[RangeFunction, T] = Map.empty ++ lower.toList ++ upper.toList - } + upper: Option[(RangeFunction.UpperBound, T)]) object RangeBound { import RangeFunction._ @@ -95,64 +130,6 @@ object query { } } - type QueryValue[T] = T :+: NonEmptyList[T] :+: CNil - type CompoundValue = QueryValue[JsonNumber] :+: QueryValue[String] :+: CNil - type RangeValue = RangeBound[JsonNumber] :+: RangeBound[String] :+: CNil - - object queryValueF extends Poly1 { - implicit def singleValue[T]: Case.Aux[T, List[T]] = at[T](List(_)) - - implicit def multipleValues[T]: Case.Aux[NonEmptyList[T], List[T]] = at[NonEmptyList[T]](_.toList) - } - - object listOfAnyValueF extends Poly1 { - implicit val caseJsonNumber: Case.Aux[QueryValue[JsonNumber], List[AnyRef]] = - at[QueryValue[JsonNumber]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) - - implicit val caseString: Case.Aux[QueryValue[String], List[AnyRef]] = - at[QueryValue[String]](_.fold(queryValueF).asInstanceOf[List[AnyRef]]) - } - - // Ugly optimisation to avoid recreation of folder instance on each implicit call - // It is safe to cast it the implicit to proper type, - // since we don't assume anything about the type, but only construct list. - private[this] val queryValueFolderInstance = new Folder[queryValueF.type, QueryValue[_]] { - type Out = List[_] - - def apply(qv: QueryValue[_]): Out = qv match { - case Inl(v) ⇒ List(v) - case Inr(Inl(vs)) ⇒ vs.toList - case _ ⇒ Nil - } - } - - // TODO: for some reason shapeless cannot find implicit `Folder` instance - // if Poly function cases contain generic param as in `queryValueF` - implicit def queryValueFolder[T]: Folder[queryValueF.type, QueryValue[T]] = - queryValueFolderInstance.asInstanceOf[Folder[queryValueF.type, QueryValue[T]]] - - implicit class RichQueryValue[T](val qv: QueryValue[T]) extends AnyVal { - def toList: List[T] = qv.fold(queryValueF).asInstanceOf[List[T]] - } - - implicit class RichCompoundValue(val cv: CompoundValue) extends AnyVal { - def toList(implicit folder: Folder[listOfAnyValueF.type, CompoundValue]): List[AnyRef] = - cv.fold(listOfAnyValueF).asInstanceOf[List[AnyRef]] - } - - implicit def decodeQueryValue[T: Decoder]: Decoder[QueryValue[T]] = - Decoder[T].map(Coproduct[QueryValue[T]](_)) or Decoder - .decodeNonEmptyList[T] - .map(Coproduct[QueryValue[T]](_)) - - implicit val decodeCompoundValue: Decoder[CompoundValue] = - Decoder[QueryValue[JsonNumber]].map(Coproduct[CompoundValue](_)) or - Decoder[QueryValue[String]].map(Coproduct[CompoundValue](_)) - - implicit val decodeRange: Decoder[RangeValue] = - Decoder[RangeBound[JsonNumber]].map(Coproduct[RangeValue](_)) or - Decoder[RangeBound[String]].map(Coproduct[RangeValue](_)) - sealed trait QueryFunction object QueryFunction { sealed trait WithField { this: QueryFunction ⇒ @@ -171,7 +148,7 @@ object query { def in: Option[QueryField] final def ctx: QueryContext = context.getOrElse(QueryContext.must) - final def field: QueryField = in.getOrElse(QueryField.Single("_all")) + final def field: QueryField = in.getOrElse(QueryField.Single(Coproduct("_all"))) } final case class matches private (in: Option[QueryField], @@ -182,24 +159,25 @@ object query { final case class equals private (in: QueryField, value: CompoundValue, context: Option[QueryContext]) extends QueryFunction with TermLevel + with WithField { + def field: QueryField = in + } final case class exists private (value: QueryField, context: Option[QueryContext]) extends QueryFunction with TermLevel final case class range private (in: QueryField.Single, value: RangeValue, context: Option[QueryContext]) extends QueryFunction with TermLevel + with WithField { + def field: QueryField.Single = in + } final case class raw private (value: JsonObject, context: QueryContext) extends QueryFunction with WithContext { def ctx: QueryContext = context } - final case class nested private (in: QueryField.Single, - context: QueryContext, - value: QueryValue[QueryFunction]) + final case class bool private (in: QueryField.Single, value: QueryValue[QueryFunction]) extends QueryFunction - with WithContext { - def ctx: QueryContext = context - } implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] } diff --git a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala index c0d0a3efba..66ece3e4d9 100644 --- a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala @@ -1,12 +1,10 @@ package foxcomm.agni.interpreter import scala.language.higherKinds -import cats.Monad import cats.data.{Kleisli, NonEmptyList} -import cats.implicits._ import foxcomm.agni.dsl.query._ -abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEmptyList[QueryFunction]), V] { +trait QueryInterpreter[F[_], V] extends Interpreter[F, (V, NonEmptyList[QueryFunction]), V] { final def kleisli: Kleisli[F, (V, NonEmptyList[QueryFunction]), V] = Kleisli(this) final def eval(v: V, qf: QueryFunction): F[V] = qf match { @@ -15,11 +13,9 @@ abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEm case qf: QueryFunction.exists ⇒ existsF(v, qf) case qf: QueryFunction.range ⇒ rangeF(v, qf) case qf: QueryFunction.raw ⇒ rawF(v, qf) - case qf: QueryFunction.nested ⇒ nestedF(v, qf) + case qf: QueryFunction.bool ⇒ boolF(v, qf) } - final def apply(v: (V, NonEmptyList[QueryFunction])): F[V] = v._2.foldM(v._1)(eval) - def matchesF(v: V, qf: QueryFunction.matches): F[V] def equalsF(v: V, qf: QueryFunction.equals): F[V] @@ -30,7 +26,7 @@ abstract class QueryInterpreter[F[_]: Monad, V] extends Interpreter[F, (V, NonEm def rawF(v: V, qf: QueryFunction.raw): F[V] - def nestedF(v: V, qf: QueryFunction.nested): F[V] + def boolF(v: V, qf: QueryFunction.bool): F[V] } object QueryInterpreter { diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala index 2c507f48cf..51a24b5f4e 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -1,5 +1,6 @@ package foxcomm.agni.interpreter.es +import cats.data.{NonEmptyList, NonEmptyVector} import cats.implicits._ import foxcomm.agni._ import foxcomm.agni.dsl.query._ @@ -9,11 +10,12 @@ import monix.cats._ import monix.eval.Coeval import org.elasticsearch.common.xcontent.{ToXContent, XContentBuilder} import org.elasticsearch.index.query._ +import shapeless.Coproduct -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) +@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.TraversableOps")) object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { - implicit class RichBoolQueryBuilder(val b: BoolQueryBuilder) extends AnyVal { - def inContext(qf: QueryFunction.WithContext)(qb: ⇒ QueryBuilder): BoolQueryBuilder = qf.ctx match { + private implicit class RichBoolQueryBuilder(val b: BoolQueryBuilder) extends AnyVal { + def inContext(qf: QueryFunction.WithContext)(qb: QueryBuilder): BoolQueryBuilder = qf.ctx match { case QueryContext.filter ⇒ b.filter(qb) case QueryContext.must ⇒ b.must(qb) case QueryContext.should ⇒ b.should(qb) @@ -21,7 +23,14 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { } } - final case class RawQueryBuilder(content: JsonObject) extends QueryBuilder { + private implicit class RichField(val f: Field) extends AnyVal { + def nest(q: String ⇒ QueryBuilder): QueryBuilder = + f.eliminate(q, + _.eliminate(fs ⇒ fs.toVector.init.foldRight(q(fs.toVector.last))(QueryBuilders.nestedQuery), + _.impossible)) + } + + private final case class RawQueryBuilder(content: JsonObject) extends QueryBuilder { def doXContent(builder: XContentBuilder, params: ToXContent.Params): Unit = { builder.startObject() content.toMap.foreach { @@ -32,51 +41,58 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { } } + def apply(v: (BoolQueryBuilder, NonEmptyList[QueryFunction])): Coeval[BoolQueryBuilder] = + v._2.foldM(v._1)(eval) + def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = Coeval.eval { - qf.value.toList.foldLeft(b)((b, v) ⇒ - b.inContext(qf) { - qf.field match { - case QueryField.Single(n) ⇒ QueryBuilders.matchQuery(n, v) - case QueryField.Multiple(ns) ⇒ - QueryBuilders.multiMatchQuery(v, ns.toList: _*) - } + qf.value.toNEL.foldLeft(b)((b, v) ⇒ + qf.field match { + case QueryField.Single(n) ⇒ b.inContext(qf)(n.nest(QueryBuilders.matchQuery(_, v))) + case QueryField.Multiple(ns) ⇒ + val (s, n) = ns.foldLeft(Vector.empty[String] → Vector.empty[NonEmptyVector[String]]) { + case ((sAcc, nAcc), f) ⇒ + f.select[String].fold(sAcc)(sAcc :+ _) → + f.select[NonEmptyVector[String]].fold(nAcc)(nAcc :+ _) + } + n.foldLeft(b.inContext(qf)(QueryBuilders.multiMatchQuery(v, s: _*)))( + (acc, f) ⇒ acc.inContext(qf)(Coproduct[Field](f).nest(QueryBuilders.matchQuery(_, v))) + ) }) } def equalsF(b: BoolQueryBuilder, qf: QueryFunction.equals): Coeval[BoolQueryBuilder] = Coeval.eval { - val vs = qf.value.toList.toList - qf.in.toList.foldLeft(b)((b, n) ⇒ + val vs = qf.value.toNEL.toList + qf.in.toNEL.foldLeft(b)((b, n) ⇒ b.inContext(qf) { vs match { - case v :: Nil ⇒ QueryBuilders.termQuery(n, v) - case _ ⇒ QueryBuilders.termsQuery(n, vs: _*) + case v :: Nil ⇒ n.nest(QueryBuilders.termQuery(_, v)) + case _ ⇒ n.nest(QueryBuilders.termsQuery(_, vs: _*)) } }) } def existsF(b: BoolQueryBuilder, qf: QueryFunction.exists): Coeval[BoolQueryBuilder] = Coeval.eval { - qf.value.toList.foldLeft(b)((b, n) ⇒ - b.inContext(qf) { - QueryBuilders.existsQuery(n) - }) + qf.value.toNEL.foldLeft(b)((b, n) ⇒ b.inContext(qf)(n.nest(QueryBuilders.existsQuery))) } def rangeF(b: BoolQueryBuilder, qf: QueryFunction.range): Coeval[BoolQueryBuilder] = Coeval.eval { b.inContext(qf) { - val builder = QueryBuilders.rangeQuery(qf.in.field) - val value = qf.value.unify - value.lower.foreach { - case (RangeFunction.Gt, v) ⇒ builder.gt(v) - case (RangeFunction.Gte, v) ⇒ builder.gte(v) - } - value.upper.foreach { - case (RangeFunction.Lt, v) ⇒ builder.lt(v) - case (RangeFunction.Lte, v) ⇒ builder.lte(v) + qf.in.field.nest { n ⇒ + val builder = QueryBuilders.rangeQuery(n) + val value = qf.value.unify + value.lower.foreach { + case (RangeFunction.Gt, v) ⇒ builder.gt(v) + case (RangeFunction.Gte, v) ⇒ builder.gte(v) + } + value.upper.foreach { + case (RangeFunction.Lt, v) ⇒ builder.lt(v) + case (RangeFunction.Lte, v) ⇒ builder.lte(v) + } + builder } - builder } } @@ -84,8 +100,6 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { b.inContext(qf)(RawQueryBuilder(qf.value)) } - def nestedF(b: BoolQueryBuilder, qf: QueryFunction.nested): Coeval[BoolQueryBuilder] = - qf.value.toList - .foldM(QueryBuilders.boolQuery())(eval) - .map(b.inContext(qf)(_)) + def boolF(b: BoolQueryBuilder, qf: QueryFunction.bool): Coeval[BoolQueryBuilder] = + qf.value.toNEL.foldM(b)(eval) } diff --git a/agni/core/app/foxcomm/agni/interpreter/es/package.scala b/agni/core/app/foxcomm/agni/interpreter/es/package.scala index 062e06cb8a..beffffe8b4 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/package.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/package.scala @@ -11,7 +11,8 @@ package object es { lazy val default: ESQueryInterpreter = { val eval: Interpreter[Coeval, NonEmptyList[QueryFunction], BoolQueryBuilder] = - ESQueryInterpreter <<< (QueryBuilders.boolQuery() → _) + (QueryBuilders.boolQuery() → (_: NonEmptyList[QueryFunction])) >>> + ESQueryInterpreter Kleisli(_.query.fold(Coeval.eval(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery())))(eval)) } diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index efef16b589..8769b69acb 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -1,5 +1,6 @@ package foxcomm.agni.dsl +import cats.data.NonEmptyVector import foxcomm.agni.dsl.query._ import io.circe.parser._ import org.scalatest.EitherValues._ @@ -10,6 +11,10 @@ import shapeless._ import shapeless.syntax.typeable._ class QueryDslSpec extends FlatSpec with Matchers { + implicit class RichRangeBound[A](val rb: RangeBound[A]) { + implicit def toMap: Map[RangeFunction, A] = Map.empty ++ rb.lower ++ rb.upper + } + def assertQueryFunction[T <: QueryFunction: Typeable](qf: QueryFunction)( assertion: T ⇒ Assertion): Assertion = assertion(qf.cast[T].value) @@ -23,15 +28,18 @@ class QueryDslSpec extends FlatSpec with Matchers { val queries = json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) assertQueryFunction[QueryFunction.equals](queries.head) { equals ⇒ - equals.in.toList should === (List("slug")) + equals.in.toList should === (List(Coproduct[Field]("slug"))) equals.value.toList should === (List("awesome", "whatever")) } assertQueryFunction[QueryFunction.matches](queries(1)) { matches ⇒ - matches.field.toList should === (List("title", "description")) + matches.field.toList should === ( + List(Coproduct[Field]("title"), + Coproduct[Field]("description"), + Coproduct[Field](NonEmptyVector.of("skus", "code")))) matches.value.toList should === (List("food", "drink")) } assertQueryFunction[QueryFunction.range](queries(2)) { range ⇒ - range.in.toList should === (List("price")) + range.in.toList should === (List(Coproduct[Field]("price"))) range.value.unify.toMap.mapValues(_.toString) should === ( Map( RangeFunction.Lt → "5000", @@ -39,7 +47,7 @@ class QueryDslSpec extends FlatSpec with Matchers { )) } assertQueryFunction[QueryFunction.exists](queries(3)) { exists ⇒ - exists.value.toList should === (List("archivedAt")) + exists.value.toList should === (List(Coproduct[Field]("archivedAt"))) exists.ctx should === (QueryContext.not) } } diff --git a/agni/core/test/resources/happy_path.json b/agni/core/test/resources/happy_path.json index 82777f2d80..46e26718fb 100644 --- a/agni/core/test/resources/happy_path.json +++ b/agni/core/test/resources/happy_path.json @@ -6,7 +6,7 @@ }, { "type": "matches", - "in": [ "title", "description" ], + "in": [ "title", "description", "skus.code" ], "value": [ "food", "drink" ] }, { From 5f7e0e7f1b6b18582877c3d02f1c992f818e8fd7 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Mon, 26 Jun 2017 11:53:00 +0200 Subject: [PATCH 39/61] Enforce more purity in QueryInterpreter --- agni/api/app/foxcomm/agni/api/Api.scala | 3 +- .../agni/interpreter/QueryInterpreter.scala | 32 +++---- .../interpreter/es/ESQueryInterpreter.scala | 92 ++++++++++--------- .../foxcomm/agni/interpreter/es/package.scala | 7 +- agni/project/Settings.scala | 1 + 5 files changed, 71 insertions(+), 64 deletions(-) diff --git a/agni/api/app/foxcomm/agni/api/Api.scala b/agni/api/app/foxcomm/agni/api/Api.scala index 2ef502f66a..34b5a81d08 100644 --- a/agni/api/app/foxcomm/agni/api/Api.scala +++ b/agni/api/app/foxcomm/agni/api/Api.scala @@ -4,6 +4,7 @@ import com.twitter.finagle.Http import com.twitter.finagle.http.Status import com.twitter.util.Await import foxcomm.agni._ +import foxcomm.agni.interpreter.es.queryInterpreter import foxcomm.utils.finch._ import io.circe.generic.extras.auto._ import io.finch._ @@ -37,7 +38,7 @@ object Api extends App { implicit val s: Scheduler = Scheduler.global val config = AppConfig.load() - val svc = SearchService.fromConfig(config, interpreter.es.default) + val svc = SearchService.fromConfig(config, queryInterpreter) Await.result( Http.server diff --git a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala index 66ece3e4d9..9f937f243d 100644 --- a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala @@ -1,32 +1,30 @@ package foxcomm.agni.interpreter import scala.language.higherKinds -import cats.data.{Kleisli, NonEmptyList} +import cats.data.NonEmptyList import foxcomm.agni.dsl.query._ -trait QueryInterpreter[F[_], V] extends Interpreter[F, (V, NonEmptyList[QueryFunction]), V] { - final def kleisli: Kleisli[F, (V, NonEmptyList[QueryFunction]), V] = Kleisli(this) - - final def eval(v: V, qf: QueryFunction): F[V] = qf match { - case qf: QueryFunction.matches ⇒ matchesF(v, qf) - case qf: QueryFunction.equals ⇒ equalsF(v, qf) - case qf: QueryFunction.exists ⇒ existsF(v, qf) - case qf: QueryFunction.range ⇒ rangeF(v, qf) - case qf: QueryFunction.raw ⇒ rawF(v, qf) - case qf: QueryFunction.bool ⇒ boolF(v, qf) +trait QueryInterpreter[F[_], V] extends Interpreter[F, NonEmptyList[QueryFunction], V] { + final def eval(qf: QueryFunction): F[V] = qf match { + case qf: QueryFunction.matches ⇒ matchesF(qf) + case qf: QueryFunction.equals ⇒ equalsF(qf) + case qf: QueryFunction.exists ⇒ existsF(qf) + case qf: QueryFunction.range ⇒ rangeF(qf) + case qf: QueryFunction.raw ⇒ rawF(qf) + case qf: QueryFunction.bool ⇒ boolF(qf) } - def matchesF(v: V, qf: QueryFunction.matches): F[V] + def matchesF(qf: QueryFunction.matches): F[V] - def equalsF(v: V, qf: QueryFunction.equals): F[V] + def equalsF(qf: QueryFunction.equals): F[V] - def existsF(v: V, qf: QueryFunction.exists): F[V] + def existsF(qf: QueryFunction.exists): F[V] - def rangeF(v: V, qf: QueryFunction.range): F[V] + def rangeF(qf: QueryFunction.range): F[V] - def rawF(v: V, qf: QueryFunction.raw): F[V] + def rawF(qf: QueryFunction.raw): F[V] - def boolF(v: V, qf: QueryFunction.bool): F[V] + def boolF(qf: QueryFunction.bool): F[V] } object QueryInterpreter { diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala index 51a24b5f4e..e58fe6e621 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -1,19 +1,23 @@ package foxcomm.agni.interpreter.es -import cats.data.{NonEmptyList, NonEmptyVector} +import cats.Id +import cats.data._ import cats.implicits._ import foxcomm.agni._ import foxcomm.agni.dsl.query._ import foxcomm.agni.interpreter.QueryInterpreter import io.circe.JsonObject -import monix.cats._ -import monix.eval.Coeval import org.elasticsearch.common.xcontent.{ToXContent, XContentBuilder} import org.elasticsearch.index.query._ import shapeless.Coproduct @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.TraversableOps")) -object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { +private[es] object ESQueryInterpreter + extends QueryInterpreter[Kleisli[Id, ?, BoolQueryBuilder], BoolQueryBuilder] { + type State = Kleisli[Id, BoolQueryBuilder, BoolQueryBuilder] + val State: (BoolQueryBuilder ⇒ Id[BoolQueryBuilder]) ⇒ State = + Kleisli[Id, BoolQueryBuilder, BoolQueryBuilder] + private implicit class RichBoolQueryBuilder(val b: BoolQueryBuilder) extends AnyVal { def inContext(qf: QueryFunction.WithContext)(qb: QueryBuilder): BoolQueryBuilder = qf.ctx match { case QueryContext.filter ⇒ b.filter(qb) @@ -41,47 +45,52 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { } } - def apply(v: (BoolQueryBuilder, NonEmptyList[QueryFunction])): Coeval[BoolQueryBuilder] = - v._2.foldM(v._1)(eval) - - def matchesF(b: BoolQueryBuilder, qf: QueryFunction.matches): Coeval[BoolQueryBuilder] = - Coeval.eval { - qf.value.toNEL.foldLeft(b)((b, v) ⇒ - qf.field match { - case QueryField.Single(n) ⇒ b.inContext(qf)(n.nest(QueryBuilders.matchQuery(_, v))) - case QueryField.Multiple(ns) ⇒ - val (s, n) = ns.foldLeft(Vector.empty[String] → Vector.empty[NonEmptyVector[String]]) { - case ((sAcc, nAcc), f) ⇒ - f.select[String].fold(sAcc)(sAcc :+ _) → - f.select[NonEmptyVector[String]].fold(nAcc)(nAcc :+ _) - } - n.foldLeft(b.inContext(qf)(QueryBuilders.multiMatchQuery(v, s: _*)))( - (acc, f) ⇒ acc.inContext(qf)(Coproduct[Field](f).nest(QueryBuilders.matchQuery(_, v))) - ) - }) - } + def apply(qfs: NonEmptyList[QueryFunction]): State = State { b ⇒ + qfs.foldM(b)((b, qf) ⇒ eval(qf)(b): Id[BoolQueryBuilder]) + } - def equalsF(b: BoolQueryBuilder, qf: QueryFunction.equals): Coeval[BoolQueryBuilder] = - Coeval.eval { - val vs = qf.value.toNEL.toList - qf.in.toNEL.foldLeft(b)((b, n) ⇒ - b.inContext(qf) { - vs match { - case v :: Nil ⇒ n.nest(QueryBuilders.termQuery(_, v)) - case _ ⇒ n.nest(QueryBuilders.termsQuery(_, vs: _*)) + def matchesF(qf: QueryFunction.matches): State = State { b ⇒ + val inContext = b.inContext(qf) _ + for (v ← qf.value.toList) { + qf.field match { + case QueryField.Single(n) ⇒ inContext(n.nest(QueryBuilders.matchQuery(_, v))) + case QueryField.Multiple(ns) ⇒ + val (sfs, nfs) = ns.foldLeft(Vector.empty[String] → Vector.empty[NonEmptyVector[String]]) { + case ((sAcc, nAcc), f) ⇒ + f.select[String].fold(sAcc)(sAcc :+ _) → + f.select[NonEmptyVector[String]].fold(nAcc)(nAcc :+ _) } - }) + inContext(QueryBuilders.multiMatchQuery(v, sfs: _*)) + nfs.foreach(nf ⇒ inContext(Coproduct[Field](nf).nest(QueryBuilders.matchQuery(_, v)))) + } } + b + } - def existsF(b: BoolQueryBuilder, qf: QueryFunction.exists): Coeval[BoolQueryBuilder] = - Coeval.eval { - qf.value.toNEL.foldLeft(b)((b, n) ⇒ b.inContext(qf)(n.nest(QueryBuilders.existsQuery))) + def equalsF(qf: QueryFunction.equals): State = State { b ⇒ + val inContext = b.inContext(qf) _ + val vs = qf.value.toList + for (f ← qf.in.toList) { + inContext { + vs match { + case v :: Nil ⇒ f.nest(QueryBuilders.termQuery(_, v)) + case _ ⇒ f.nest(QueryBuilders.termsQuery(_, vs: _*)) + } + } } + b + } - def rangeF(b: BoolQueryBuilder, qf: QueryFunction.range): Coeval[BoolQueryBuilder] = Coeval.eval { + def existsF(qf: QueryFunction.exists): State = State { b ⇒ + val inContext = b.inContext(qf) _ + qf.value.toList.foreach(f ⇒ inContext(f.nest(QueryBuilders.existsQuery))) + b + } + + def rangeF(qf: QueryFunction.range): State = State { b ⇒ b.inContext(qf) { - qf.in.field.nest { n ⇒ - val builder = QueryBuilders.rangeQuery(n) + qf.in.field.nest { f ⇒ + val builder = QueryBuilders.rangeQuery(f) val value = qf.value.unify value.lower.foreach { case (RangeFunction.Gt, v) ⇒ builder.gt(v) @@ -96,10 +105,11 @@ object ESQueryInterpreter extends QueryInterpreter[Coeval, BoolQueryBuilder] { } } - def rawF(b: BoolQueryBuilder, qf: QueryFunction.raw): Coeval[BoolQueryBuilder] = Coeval.eval { + def rawF(qf: QueryFunction.raw): State = State { b ⇒ b.inContext(qf)(RawQueryBuilder(qf.value)) } - def boolF(b: BoolQueryBuilder, qf: QueryFunction.bool): Coeval[BoolQueryBuilder] = - qf.value.toNEL.foldM(b)(eval) + def boolF(qf: QueryFunction.bool): State = State { b ⇒ + qf.value.toNEL.foldM(b)((b, qf) ⇒ eval(qf)(b): Id[BoolQueryBuilder]) + } } diff --git a/agni/core/app/foxcomm/agni/interpreter/es/package.scala b/agni/core/app/foxcomm/agni/interpreter/es/package.scala index beffffe8b4..21ce4b006a 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/package.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/package.scala @@ -1,7 +1,6 @@ package foxcomm.agni.interpreter import cats.data._ -import cats.implicits._ import foxcomm.agni.dsl.query.{FCQuery, QueryFunction} import monix.eval.Coeval import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} @@ -9,11 +8,9 @@ import org.elasticsearch.index.query.{BoolQueryBuilder, QueryBuilders} package object es { type ESQueryInterpreter = Kleisli[Coeval, FCQuery, BoolQueryBuilder] - lazy val default: ESQueryInterpreter = { + val queryInterpreter: ESQueryInterpreter = { val eval: Interpreter[Coeval, NonEmptyList[QueryFunction], BoolQueryBuilder] = - (QueryBuilders.boolQuery() → (_: NonEmptyList[QueryFunction])) >>> - ESQueryInterpreter - + ESQueryInterpreter andThen (f ⇒ Coeval.eval(f(QueryBuilders.boolQuery()))) Kleisli(_.query.fold(Coeval.eval(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery())))(eval)) } } diff --git a/agni/project/Settings.scala b/agni/project/Settings.scala index 66f7a4cd9a..488334aaa2 100644 --- a/agni/project/Settings.scala +++ b/agni/project/Settings.scala @@ -26,6 +26,7 @@ object Settings { "-Xfatal-warnings", "-Xfuture" ), + addCompilerPlugin("org.spire-math" %% "kind-projector" % "0.9.4"), wartremoverErrors in (Compile, compile) ++= Warts.allBut(Wart.Any, Wart.ImplicitParameter, Wart.Nothing, From 555b4cb42fd1f55018cb0dc6b49cdb99a1a3c23b Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 26 Jun 2017 14:22:52 +0300 Subject: [PATCH 40/61] Change nginx proxy settings + minor improvements --- Makefile | 2 +- .../balancer/templates/service_locations.j2 | 7 ++---- .../templates/core-backend/agni.json.j2 | 3 ++- .../dev/nginx/templates/service_locations.j2 | 23 ++++++++----------- .../roles/dev/nginx/templates/services.j2 | 1 + .../hotfix/drop_mat_views/tasks/main.yml | 8 +++---- 6 files changed, 20 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 5f60274c0a..703cbc9864 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ config: up: $(call header, Creating Appliance) - ansible-playbook -vvvv --user=$(GOOGLE_SSH_USERNAME) --private-key=$(GOOGLE_SSH_KEY) --extra-vars '{"FIRST_RUN": false}' tabernacle/ansible/goldrush_appliance.yml + ansible-playbook -vvvv --user=$(GOOGLE_SSH_USERNAME) --private-key=$(GOOGLE_SSH_KEY) --extra-vars '{"FIRST_RUN": true}' tabernacle/ansible/goldrush_appliance.yml @cat goldrush.log destroy: diff --git a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 index 6c30d0e4af..20179048a5 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 @@ -145,12 +145,9 @@ location ~ /api/search/public/.*/\d+$ { } # Proxy to agni -location /api/advanced-search/.*/\d+$ { - auth_request /internal-auth; - default_type 'application/json'; - proxy_pass http://agni/search/; +location /api/advanced-search/ { + proxy_pass http://agni/api/search/; proxy_http_version 1.1; - proxy_set_header Connection ""; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index 3121e8294f..55d148f44d 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -19,7 +19,8 @@ }, "env": { "PORT": "{{agni_port}}", - "SEARCH_SERVER": "elasticsearch://{{search_server}}" + "SEARCH_SERVER": "elasticsearch://{{search_server}}", + "JAVA_OPTS":"" }, "ports": [{{agni_port}}], "healthChecks": [{ diff --git a/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 index 4c41d6337a..f245db87da 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/service_locations.j2 @@ -121,19 +121,6 @@ location ~ /api/search/public/.*/_count { break; } -# Proxy to agni -location /api/advanced-search/.*/\d+$ { - auth_request /internal-auth; - default_type 'application/json'; - proxy_pass http://agni/search/; - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - break; -} - # Proxy the internal search location and sanitizes output from ES for external use location ~ /api/search/public/.*/\d+$ { default_type 'application/json'; @@ -157,6 +144,16 @@ location ~ /api/search/public/.*/\d+$ { break; } +# Proxy to agni +location /api/advanced-search/ { + proxy_pass http://agni/api/search/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + break; +} + # Proxy to middlewarehouse location /api/v1/inventory/ { auth_request /internal-auth; diff --git a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 index 4b37dbb5a0..6f719b1977 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 @@ -66,6 +66,7 @@ upstream hyperion { upstream geronimo { << range service "geronimo" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; << else >> server {{geronimo_server}} fail_timeout=30s max_fails=10; << end >> +} # We use port 9000 for agni upstream agni { diff --git a/tabernacle/ansible/roles/hotfix/drop_mat_views/tasks/main.yml b/tabernacle/ansible/roles/hotfix/drop_mat_views/tasks/main.yml index 5194db1093..7fc2092c48 100644 --- a/tabernacle/ansible/roles/hotfix/drop_mat_views/tasks/main.yml +++ b/tabernacle/ansible/roles/hotfix/drop_mat_views/tasks/main.yml @@ -1,14 +1,14 @@ --- -- name: Stop and delete service +- name: Stop and delete materialized_views service service: name=materialized_views state=stopped enabled=no -- name: Delete files which belongs to obsoleted materialized services +- name: Delete all the files associated with that service file: path={{item}} state=absent with_items: - "{{usr_local}}/bin/update_materialized_views_forever.sh" - "/usr/local/share/update_materialized_views.sql" - "{{dest_services}}/materialized_views.service" -- name: Reload services - systemd: daemon_reload=yes +- name: Reload daemon + command: systemctl daemon-reload From 1846cd8871d2e182ac34c431a7738e5f08167552 Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 26 Jun 2017 14:33:22 +0300 Subject: [PATCH 41/61] Add hotfixes --- tabernacle/ansible/goldrush_appliance.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tabernacle/ansible/goldrush_appliance.yml b/tabernacle/ansible/goldrush_appliance.yml index d4681e233d..cdf75e4390 100644 --- a/tabernacle/ansible/goldrush_appliance.yml +++ b/tabernacle/ansible/goldrush_appliance.yml @@ -30,6 +30,8 @@ api_server: "appliance-{{ansible_default_ipv4.address | replace(\".\", \"-\") }}.foxcommerce.com" roles: - { role: dev/dashboard, when: first_run } + - { role: hotfix/rsyslog, when: first_run } + - { role: hotfix/drop_mat_views, when: first_run } - { role: dev/flyway } - { role: dev/seed_system, when: first_run } - { role: dev/marathon_groups } From b6b192d471139de2b19d0b7ea4892cb3b70ace17 Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 26 Jun 2017 17:16:37 +0300 Subject: [PATCH 42/61] Testing buildkite --- tabernacle/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tabernacle/Makefile b/tabernacle/Makefile index 7cdfba9261..f0b49fe14a 100644 --- a/tabernacle/Makefile +++ b/tabernacle/Makefile @@ -1,6 +1,8 @@ include ../makelib header = $(call baseheader, $(1), tabernacle) +TEST_VAR = false + GO ?= go GOPATH := $(CURDIR)/_vendor:$(GOPATH) From 37757792492cb4846bb4fe46530360d1d5e804fa Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 26 Jun 2017 17:21:44 +0300 Subject: [PATCH 43/61] Revert "Testing buildkite" This reverts commit b6b192d471139de2b19d0b7ea4892cb3b70ace17. --- tabernacle/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/tabernacle/Makefile b/tabernacle/Makefile index f0b49fe14a..7cdfba9261 100644 --- a/tabernacle/Makefile +++ b/tabernacle/Makefile @@ -1,8 +1,6 @@ include ../makelib header = $(call baseheader, $(1), tabernacle) -TEST_VAR = false - GO ?= go GOPATH := $(CURDIR)/_vendor:$(GOPATH) From 8aa736091f9c1b6dcc81e8d3b47e75fbfd8b79d4 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Mon, 26 Jun 2017 20:45:52 +0200 Subject: [PATCH 44/61] Support boost per context --- agni/core/app/foxcomm/agni/dsl/query.scala | 72 ++++++++++++++----- .../agni/interpreter/QueryInterpreter.scala | 19 +++-- .../interpreter/es/ESQueryInterpreter.scala | 20 ++++-- .../test/foxcomm/agni/dsl/QueryDslSpec.scala | 13 +++- agni/core/test/resources/happy_path.json | 2 + 5 files changed, 92 insertions(+), 34 deletions(-) diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 14b686f658..80461cf2e3 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -1,17 +1,21 @@ package foxcomm.agni.dsl +import scala.language.higherKinds import cats.data.{NonEmptyList, NonEmptyVector} import cats.implicits._ +import cats.instances.either._ +import io.circe.Decoder.Result import io.circe._ import io.circe.generic.extras.semiauto._ +import scala.util.Try import shapeless._ -@SuppressWarnings(Array("org.wartremover.warts.Equals")) object query { - type QueryValue[T] = T :+: NonEmptyList[T] :+: CNil - type CompoundValue = QueryValue[JsonNumber] :+: QueryValue[String] :+: CNil - type Field = String :+: NonEmptyVector[String] :+: CNil - type RangeValue = RangeBound[JsonNumber] :+: RangeBound[String] :+: CNil + type QueryValueF[F[_], T] = T :+: F[T] :+: CNil + type QueryValue[T] = QueryValueF[NonEmptyList, T] + type CompoundValue = QueryValue[JsonNumber] :+: QueryValue[String] :+: CNil + type Field = QueryValueF[NonEmptyVector, String] + type RangeValue = RangeBound[JsonNumber] :+: RangeBound[String] :+: CNil implicit class RichQueryValue[T](val qv: QueryValue[T]) extends AnyVal { def toNEL: NonEmptyList[T] = qv.eliminate(NonEmptyList.of(_), _.eliminate(identity, _.impossible)) @@ -25,10 +29,9 @@ object query { def toList: List[AnyRef] = toNEL.toList } - implicit def decodeQueryValue[T: Decoder]: Decoder[QueryValue[T]] = - Decoder[T].map(Coproduct[QueryValue[T]](_)) or Decoder - .decodeNonEmptyList[T] - .map(Coproduct[QueryValue[T]](_)) + implicit def decodeQueryValueF[F[_], T](implicit fD: Decoder[F[T]], + tD: Decoder[T]): Decoder[QueryValueF[F, T]] = + tD.map(Coproduct[QueryValueF[F, T]](_)) or fD.map(Coproduct[QueryValueF[F, T]](_)) implicit val decodeCompoundValue: Decoder[CompoundValue] = Decoder[QueryValue[JsonNumber]].map(Coproduct[CompoundValue](_)) or @@ -44,12 +47,36 @@ object query { Decoder[RangeBound[JsonNumber]].map(Coproduct[RangeValue](_)) or Decoder[RangeBound[String]].map(Coproduct[RangeValue](_)) + object Boostable { + private[this] val explicitBoostableRegex = "^(\\w+)\\^([0-9]*\\.?[0-9]+)$".r + private[this] val boostableRegex = "^(\\w+)\\^$".r + + object explicit { + def unapply(s: String): Option[(String, Float)] = s match { + case explicitBoostableRegex(f, b) ⇒ Try(f → b.toFloat).toOption + case _ ⇒ None + } + } + + private[this] val explicitMatcher = (explicit.unapply _).andThen(_.map { case (f, b) ⇒ f → Some(b) }) + + def unapply(s: String): Option[(String, Option[Float])] = + explicitMatcher(s) orElse (s match { + case boostableRegex(f) ⇒ Some(f → Some(default)) + case f ⇒ Some(f → None) + }) + + def default: Float = 1.0f + } + sealed trait QueryField { def toNEL: NonEmptyList[Field] def toList: List[Field] = toNEL.toList } object QueryField { + final case class Value(field: String, boost: Option[Float]) + final case class Single(field: Field) extends QueryField { def toNEL: NonEmptyList[Field] = NonEmptyList.of(field) } @@ -72,12 +99,19 @@ object query { sealed trait QueryContext object QueryContext { - case object filter extends QueryContext - case object must extends QueryContext - case object should extends QueryContext - case object not extends QueryContext - - implicit val decodeQueryContext: Decoder[QueryContext] = deriveEnumerationDecoder[QueryContext] + final case class must(boost: Option[Float]) extends QueryContext + final case class should(boost: Option[Float]) extends QueryContext + final case class not(boost: Option[Float]) extends QueryContext + + implicit val decodeQueryContext: Decoder[QueryContext] = new Decoder[QueryContext] { + def apply(c: HCursor): Result[QueryContext] = c.as[String].flatMap { + case Boostable("must", b) ⇒ Either.right(must(b)) + case Boostable("should", b) ⇒ Either.right(should(b)) + case Boostable.explicit("not", b) ⇒ Either.right(not(Some(b))) + case "not" ⇒ Either.right(not(None)) + case _ ⇒ Either.left(DecodingFailure("invalid query context", c.history)) + } + } } sealed trait RangeFunction @@ -124,8 +158,8 @@ object query { case (ub: UpperBound, v) ⇒ ub → v }.toList - if (lbs.size > 1) Either.left("Only single lower bound can be specified") - else if (ubs.size > 1) Either.left("Only single upper bound can be specified") + if (lbs.size > 1) Either.left("only single lower bound can be specified") + else if (ubs.size > 1) Either.left("only single upper bound can be specified") else Either.right(RangeBound(lbs.headOption, ubs.headOption)) } } @@ -141,13 +175,13 @@ object query { sealed trait TermLevel extends WithContext { this: QueryFunction ⇒ def context: Option[QueryContext] - final def ctx: QueryContext = context.getOrElse(QueryContext.filter) + final def ctx: QueryContext = context.getOrElse(QueryContext.must(None)) } sealed trait FullText extends WithContext with WithField { this: QueryFunction ⇒ def context: Option[QueryContext] def in: Option[QueryField] - final def ctx: QueryContext = context.getOrElse(QueryContext.must) + final def ctx: QueryContext = context.getOrElse(QueryContext.must(Some(Boostable.default))) final def field: QueryField = in.getOrElse(QueryField.Single(Coproduct("_all"))) } diff --git a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala index 9f937f243d..9af3d8445a 100644 --- a/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/QueryInterpreter.scala @@ -4,8 +4,13 @@ import scala.language.higherKinds import cats.data.NonEmptyList import foxcomm.agni.dsl.query._ +sealed trait QueryError +object QueryError {} + trait QueryInterpreter[F[_], V] extends Interpreter[F, NonEmptyList[QueryFunction], V] { - final def eval(qf: QueryFunction): F[V] = qf match { + type Result = V + + final def eval(qf: QueryFunction): F[Result] = qf match { case qf: QueryFunction.matches ⇒ matchesF(qf) case qf: QueryFunction.equals ⇒ equalsF(qf) case qf: QueryFunction.exists ⇒ existsF(qf) @@ -14,17 +19,17 @@ trait QueryInterpreter[F[_], V] extends Interpreter[F, NonEmptyList[QueryFunctio case qf: QueryFunction.bool ⇒ boolF(qf) } - def matchesF(qf: QueryFunction.matches): F[V] + def matchesF(qf: QueryFunction.matches): F[Result] - def equalsF(qf: QueryFunction.equals): F[V] + def equalsF(qf: QueryFunction.equals): F[Result] - def existsF(qf: QueryFunction.exists): F[V] + def existsF(qf: QueryFunction.exists): F[Result] - def rangeF(qf: QueryFunction.range): F[V] + def rangeF(qf: QueryFunction.range): F[Result] - def rawF(qf: QueryFunction.raw): F[V] + def rawF(qf: QueryFunction.raw): F[Result] - def boolF(qf: QueryFunction.bool): F[V] + def boolF(qf: QueryFunction.bool): F[Result] } object QueryInterpreter { diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala index e58fe6e621..71b6412a4d 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -19,11 +19,19 @@ private[es] object ESQueryInterpreter Kleisli[Id, BoolQueryBuilder, BoolQueryBuilder] private implicit class RichBoolQueryBuilder(val b: BoolQueryBuilder) extends AnyVal { - def inContext(qf: QueryFunction.WithContext)(qb: QueryBuilder): BoolQueryBuilder = qf.ctx match { - case QueryContext.filter ⇒ b.filter(qb) - case QueryContext.must ⇒ b.must(qb) - case QueryContext.should ⇒ b.should(qb) - case QueryContext.not ⇒ b.mustNot(qb) + def inContext(qf: QueryFunction.WithContext)(qb: QueryBuilder): Unit = qf.ctx match { + case QueryContext.must(None) ⇒ b.filter(qb) + case QueryContext.must(Some(boost)) ⇒ b.must(qb).boost(boost) + case QueryContext.should(boost) ⇒ b.should(qb).boost(boost.getOrElse(Boostable.default)) + case QueryContext.not(None) ⇒ b.mustNot(qb) + case QueryContext.not(Some(boost)) ⇒ + val boosting = QueryBuilders + .boostingQuery() + .negativeBoost(boost) + .negative(qb) + .boost(Boostable.default) + .positive(QueryBuilders.matchAllQuery()) + b.must(boosting) } } @@ -103,10 +111,12 @@ private[es] object ESQueryInterpreter builder } } + b } def rawF(qf: QueryFunction.raw): State = State { b ⇒ b.inContext(qf)(RawQueryBuilder(qf.value)) + b } def boolF(qf: QueryFunction.bool): State = State { b ⇒ diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index 8769b69acb..9b6c810d88 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -28,7 +28,9 @@ class QueryDslSpec extends FlatSpec with Matchers { val queries = json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) assertQueryFunction[QueryFunction.equals](queries.head) { equals ⇒ - equals.in.toList should === (List(Coproduct[Field]("slug"))) + equals.field.toList should === (List(Coproduct[Field]("slug"))) + equals.ctx should === (QueryContext.must(Some(Boostable.default))) + equals.context should be('defined) equals.value.toList should === (List("awesome", "whatever")) } assertQueryFunction[QueryFunction.matches](queries(1)) { matches ⇒ @@ -36,10 +38,14 @@ class QueryDslSpec extends FlatSpec with Matchers { List(Coproduct[Field]("title"), Coproduct[Field]("description"), Coproduct[Field](NonEmptyVector.of("skus", "code")))) + matches.ctx should === (QueryContext.should(Some(0.5f))) + matches.context should be('defined) matches.value.toList should === (List("food", "drink")) } assertQueryFunction[QueryFunction.range](queries(2)) { range ⇒ - range.in.toList should === (List(Coproduct[Field]("price"))) + range.field.toList should === (List(Coproduct[Field]("price"))) + range.ctx should === (QueryContext.must(None)) + range.context should be('empty) range.value.unify.toMap.mapValues(_.toString) should === ( Map( RangeFunction.Lt → "5000", @@ -48,7 +54,8 @@ class QueryDslSpec extends FlatSpec with Matchers { } assertQueryFunction[QueryFunction.exists](queries(3)) { exists ⇒ exists.value.toList should === (List(Coproduct[Field]("archivedAt"))) - exists.ctx should === (QueryContext.not) + exists.ctx should === (QueryContext.not(None)) + exists.context should be('defined) } } } diff --git a/agni/core/test/resources/happy_path.json b/agni/core/test/resources/happy_path.json index 46e26718fb..1262d455b9 100644 --- a/agni/core/test/resources/happy_path.json +++ b/agni/core/test/resources/happy_path.json @@ -1,11 +1,13 @@ [ { "type": "equals", + "context": "must^", "in": "slug", "value": [ "awesome", "whatever" ] }, { "type": "matches", + "context": "should^.5", "in": [ "title", "description", "skus.code" ], "value": [ "food", "drink" ] }, From d30d799a4e128cf947fb3325eaf5620e1540969c Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 27 Jun 2017 12:23:15 +0200 Subject: [PATCH 45/61] Add max depth check for bool query --- agni/core/app/foxcomm/agni/dsl/query.scala | 70 +++++++++++++------ .../test/foxcomm/agni/dsl/QueryDslSpec.scala | 42 ++++++++++- .../{happy_path.json => query/multiple.json} | 17 +++++ 3 files changed, 106 insertions(+), 23 deletions(-) rename agni/core/test/resources/{happy_path.json => query/multiple.json} (62%) diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 80461cf2e3..5e362d915b 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -4,12 +4,12 @@ import scala.language.higherKinds import cats.data.{NonEmptyList, NonEmptyVector} import cats.implicits._ import cats.instances.either._ -import io.circe.Decoder.Result import io.circe._ import io.circe.generic.extras.semiauto._ import scala.util.Try import shapeless._ +@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) object query { type QueryValueF[F[_], T] = T :+: F[T] :+: CNil type QueryValue[T] = QueryValueF[NonEmptyList, T] @@ -93,8 +93,7 @@ object query { } implicit val decodeQueryField: Decoder[QueryField] = - Decoder[Single].map(identity[QueryField]) or - Decoder[Multiple].map(identity[QueryField]) + Decoder[Single].map(identity) or Decoder[Multiple].map(identity) } sealed trait QueryContext @@ -103,15 +102,14 @@ object query { final case class should(boost: Option[Float]) extends QueryContext final case class not(boost: Option[Float]) extends QueryContext - implicit val decodeQueryContext: Decoder[QueryContext] = new Decoder[QueryContext] { - def apply(c: HCursor): Result[QueryContext] = c.as[String].flatMap { + implicit val decodeQueryContext: Decoder[QueryContext] = Decoder.instance[QueryContext](c ⇒ + c.as[String].flatMap { case Boostable("must", b) ⇒ Either.right(must(b)) case Boostable("should", b) ⇒ Either.right(should(b)) case Boostable.explicit("not", b) ⇒ Either.right(not(Some(b))) case "not" ⇒ Either.right(not(None)) - case _ ⇒ Either.left(DecodingFailure("invalid query context", c.history)) - } - } + case _ ⇒ Either.left(DecodingFailure("Invalid query context", c.history)) + }) } sealed trait RangeFunction @@ -158,26 +156,29 @@ object query { case (ub: UpperBound, v) ⇒ ub → v }.toList - if (lbs.size > 1) Either.left("only single lower bound can be specified") - else if (ubs.size > 1) Either.left("only single upper bound can be specified") + if (lbs.size > 1) Either.left("Only single lower bound can be specified") + else if (ubs.size > 1) Either.left("Only single upper bound can be specified") else Either.right(RangeBound(lbs.headOption, ubs.headOption)) } } sealed trait QueryFunction object QueryFunction { - sealed trait WithField { this: QueryFunction ⇒ + sealed trait Basic extends QueryFunction + + sealed trait WithField extends Basic { this: QueryFunction ⇒ def field: QueryField } - sealed trait WithContext { this: QueryFunction ⇒ + sealed trait WithContext extends Basic { this: QueryFunction ⇒ def ctx: QueryContext } - sealed trait TermLevel extends WithContext { this: QueryFunction ⇒ + + sealed trait TermLevel extends Basic with WithContext { this: QueryFunction ⇒ def context: Option[QueryContext] final def ctx: QueryContext = context.getOrElse(QueryContext.must(None)) } - sealed trait FullText extends WithContext with WithField { this: QueryFunction ⇒ + sealed trait FullText extends Basic with WithContext with WithField { this: QueryFunction ⇒ def context: Option[QueryContext] def in: Option[QueryField] @@ -190,30 +191,60 @@ object query { context: Option[QueryContext]) extends QueryFunction with FullText + final case class equals private (in: QueryField, value: CompoundValue, context: Option[QueryContext]) extends QueryFunction with TermLevel with WithField { def field: QueryField = in } + final case class exists private (value: QueryField, context: Option[QueryContext]) extends QueryFunction with TermLevel + final case class range private (in: QueryField.Single, value: RangeValue, context: Option[QueryContext]) extends QueryFunction with TermLevel with WithField { def field: QueryField.Single = in } + final case class raw private (value: JsonObject, context: QueryContext) extends QueryFunction with WithContext { def ctx: QueryContext = context } - final case class bool private (in: QueryField.Single, value: QueryValue[QueryFunction]) - extends QueryFunction - implicit val decodeQueryFunction: Decoder[QueryFunction] = deriveDecoder[QueryFunction] + final case class bool private (value: QueryValue[QueryFunction]) extends QueryFunction + object bool { + // TODO: make it configurable (?) + val MaxDepth = 25 + + implicit val decodeBool: Decoder[bool] = { + val decoder = deriveDecoder[bool] + val depthField = "_depth" + + Decoder.instance { c ⇒ + val depth = (for { + parent ← c.up.focus + parent ← parent.asObject + depth ← parent(depthField) + depth ← depth.as[Int].toOption + } yield depth).getOrElse(1) + + // we start counting from 0, + // which denotes implicit top-level bool query + if (depth >= MaxDepth) + Either.left(DecodingFailure(s"Max depth of $MaxDepth exceeded for a bool query", c.history)) + else + decoder.tryDecode(c.withFocus(_.mapObject(_.add(depthField, Json.fromInt(depth + 1))))) + } + } + } + + implicit val decodeQueryFunction: Decoder[QueryFunction] = + deriveDecoder[QueryFunction.Basic].map(identity) or Decoder[bool].map(identity) } final case class FCQuery(query: Option[NonEmptyList[QueryFunction]]) @@ -221,9 +252,8 @@ object query { implicit val decodeFCQuery: Decoder[FCQuery] = { Decoder .decodeOption( - Decoder - .decodeNonEmptyList[QueryFunction] - .or(Decoder[QueryFunction].map(NonEmptyList.of(_)))) + Decoder.decodeNonEmptyList[QueryFunction] or + Decoder[QueryFunction].map(NonEmptyList.of(_))) .map(FCQuery(_)) } } diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index 9b6c810d88..94515af8f6 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -2,10 +2,11 @@ package foxcomm.agni.dsl import cats.data.NonEmptyVector import foxcomm.agni.dsl.query._ +import io.circe.{Json, JsonObject} import io.circe.parser._ import org.scalatest.EitherValues._ -import org.scalatest.OptionValues._ import org.scalatest.{Assertion, FlatSpec, Matchers} +import scala.annotation.tailrec import scala.io.Source import shapeless._ import shapeless.syntax.typeable._ @@ -17,13 +18,14 @@ class QueryDslSpec extends FlatSpec with Matchers { def assertQueryFunction[T <: QueryFunction: Typeable](qf: QueryFunction)( assertion: T ⇒ Assertion): Assertion = - assertion(qf.cast[T].value) + qf.cast[T] + .fold(fail(s"Cannot cast query function ${qf.getClass.getName} to ${Typeable[T].describe}"))(assertion) "DSL" should "parse multiple queries" in { val json = parse( Source - .fromInputStream(getClass.getResourceAsStream("/happy_path.json")) + .fromInputStream(getClass.getResourceAsStream("/query/multiple.json")) .mkString).right.value val queries = json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) @@ -57,5 +59,39 @@ class QueryDslSpec extends FlatSpec with Matchers { exists.ctx should === (QueryContext.not(None)) exists.context should be('defined) } + assertQueryFunction[QueryFunction.bool](queries(4)) { bool ⇒ + val qfs = bool.value.toNEL + assertQueryFunction[QueryFunction.equals](qfs.head) { equals ⇒ + equals.field.toList should === (List(Coproduct[Field]("context"))) + equals.value.toList should === (List("default")) + } + assertQueryFunction[QueryFunction.bool](qfs.tail.head) { bool ⇒ + assertQueryFunction[QueryFunction.exists](bool.value.toNEL.head) { exists ⇒ + exists.value.toList should === (List(Coproduct[Field]("context"))) + } + } + } + } + + it should "limit max depth for bool query" in { + val leaf = JsonObject.fromMap( + Map( + "type" → Json.fromString("exists"), + "value" → Json.arr(Json.fromString("answer"), Json.fromString("to"), Json.fromString("everything")) + )) + + @tailrec + def deepBool(boolDepth: Int, embed: JsonObject): Json = + if (boolDepth > 0) + deepBool(boolDepth - 1, + JsonObject.fromMap( + Map( + "type" → Json.fromString("bool"), + "value" → Json.fromJsonObject(embed) + ))) + else Json.fromJsonObject(embed) + + deepBool(boolDepth = QueryFunction.bool.MaxDepth - 1, embed = leaf).as[FCQuery].isLeft should === (false) + deepBool(boolDepth = QueryFunction.bool.MaxDepth, embed = leaf).as[FCQuery].isLeft should === (true) } } diff --git a/agni/core/test/resources/happy_path.json b/agni/core/test/resources/query/multiple.json similarity index 62% rename from agni/core/test/resources/happy_path.json rename to agni/core/test/resources/query/multiple.json index 1262d455b9..fe009a03b5 100644 --- a/agni/core/test/resources/happy_path.json +++ b/agni/core/test/resources/query/multiple.json @@ -23,5 +23,22 @@ "type": "exists", "context": "not", "value": "archivedAt" + }, + { + "type": "bool", + "value": [ + { + "type": "equals", + "in": "context", + "value": "default" + }, + { + "type": "bool", + "value": { + "type": "exists", + "value": "context" + } + } + ] } ] From e0f7e2ec94eb460ab5788551dfca3289fb0d35d8 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Tue, 27 Jun 2017 12:48:40 +0200 Subject: [PATCH 46/61] Add translate endpoint --- agni/api/app/foxcomm/agni/api/Api.scala | 10 +++++-- .../core/app/foxcomm/agni/SearchService.scala | 27 ++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/agni/api/app/foxcomm/agni/api/Api.scala b/agni/api/app/foxcomm/agni/api/Api.scala index 34b5a81d08..621f86b0dd 100644 --- a/agni/api/app/foxcomm/agni/api/Api.scala +++ b/agni/api/app/foxcomm/agni/api/Api.scala @@ -4,6 +4,7 @@ import com.twitter.finagle.Http import com.twitter.finagle.http.Status import com.twitter.util.Await import foxcomm.agni._ +import foxcomm.agni.dsl.query._ import foxcomm.agni.interpreter.es.queryInterpreter import foxcomm.utils.finch._ import io.circe.generic.extras.auto._ @@ -14,14 +15,19 @@ import org.elasticsearch.common.ValidationException object Api extends App { def endpoints(searchService: SearchService)(implicit s: Scheduler) = - post( + post("api" :: "search" :: "translate" :: jsonBody[SearchPayload.fc]) { (searchPayload: SearchPayload.fc) ⇒ + searchService + .translate(searchPayload = searchPayload) + .map(Ok) + .toTwitterFuture + } :+: post( "api" :: "search" :: string :: string :: param("size") .as[Int] :: paramOption("from").as[Int] :: jsonBody[SearchPayload]) { (searchIndex: String, searchType: String, size: Int, from: Option[Int], searchQuery: SearchPayload) ⇒ searchService .searchFor(searchIndex = searchIndex, searchType = searchType, - searchQuery = searchQuery, + searchPayload = searchQuery, searchSize = size, searchFrom = from) .map(Ok) diff --git a/agni/core/app/foxcomm/agni/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala index 6c9c3fb3e3..105a608650 100644 --- a/agni/core/app/foxcomm/agni/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -10,15 +10,36 @@ import org.elasticsearch.client.Client import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.transport.InetSocketTransportAddress +import org.elasticsearch.common.xcontent.{ToXContent, XContentFactory} +import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.SearchHit @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) class SearchService private (client: Client, qi: ESQueryInterpreter) { import SearchService.ExtractJsonObject + def translate(searchPayload: SearchPayload.fc): Task[Json] = { + def buildJson(qb: QueryBuilder): Coeval[Json] = + Coeval.eval { + val builder = XContentFactory.jsonBuilder() + builder.prettyPrint() + builder.startObject() + builder.field("query") + qb.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.endObject() + parseByteBuffer(builder.bytes().toChannelBuffer.toByteBuffer) + .fold(Coeval.raiseError(_), Coeval.eval(_)) + }.flatten + + for { + builder ← qi(searchPayload.query).task + json ← buildJson(builder).task + } yield json + } + def searchFor(searchIndex: String, searchType: String, - searchQuery: SearchPayload, + searchPayload: SearchPayload, searchSize: Int, searchFrom: Option[Int]): Task[SearchResult] = { def prepareBuilder: Coeval[SearchRequestBuilder] = Coeval.eval { @@ -28,11 +49,11 @@ class SearchService private (client: Client, qi: ESQueryInterpreter) { .setTypes(searchType) .setSize(searchSize) searchFrom.foreach(builder.setFrom) - searchQuery.fields.foreach(fs ⇒ builder.setFetchSource(fs.toList.toArray, Array.empty[String])) + searchPayload.fields.foreach(fs ⇒ builder.setFetchSource(fs.toList.toArray, Array.empty[String])) builder } - def evalQuery(builder: SearchRequestBuilder): Coeval[SearchRequestBuilder] = searchQuery match { + def evalQuery(builder: SearchRequestBuilder): Coeval[SearchRequestBuilder] = searchPayload match { case SearchPayload.es(query, _) ⇒ Coeval.eval(builder.setQuery(Json.fromJsonObject(query).dump)) case SearchPayload.fc(query, _) ⇒ From d4323372607d4c1db232ad4961e3303c45d5037a Mon Sep 17 00:00:00 2001 From: Alexey Afanasev Date: Wed, 28 Jun 2017 16:35:00 +0300 Subject: [PATCH 47/61] Simplify spawning local agni instance --- agni/.env.local.sample | 2 ++ agni/Makefile | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 agni/.env.local.sample diff --git a/agni/.env.local.sample b/agni/.env.local.sample new file mode 100644 index 0000000000..90e71fb376 --- /dev/null +++ b/agni/.env.local.sample @@ -0,0 +1,2 @@ +export SEARCH_SERVER=elasticsearch://10.240.0.18:9300 +export PORT=9000 diff --git a/agni/Makefile b/agni/Makefile index 5eb977c213..6f18763659 100644 --- a/agni/Makefile +++ b/agni/Makefile @@ -5,10 +5,16 @@ DOCKER_REPO ?= $(DOCKER_STAGE_REPO) DOCKER_IMAGE ?= agni DOCKER_TAG ?= master SBT_CMD = sbt -DDOCKER_REPO=$(DOCKER_REPO) -DDOCKER_TAG=${DOCKER_TAG} +SEARCH_SERVER ?= elasticsearch://10.240.0.18:9300 +PORT ?= 9000 autoformat-check: ../utils/scalafmt/scalafmt.sh --test +run: + $(call header, Run locally) + sbt api/run + clean: $(call header, Cleaning) ${SBT_CMD} '; clean' From 271afa127699cbc45eab095bb512e3eaeb0a94c3 Mon Sep 17 00:00:00 2001 From: Alexey Afanasev Date: Wed, 28 Jun 2017 17:25:02 +0300 Subject: [PATCH 48/61] PR suggestions addressed --- agni/Makefile | 4 +--- agni/core/resources/reference.conf | 1 + agni/project/sbt-resolver.sbt | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 agni/project/sbt-resolver.sbt diff --git a/agni/Makefile b/agni/Makefile index 6f18763659..8f83b02bf8 100644 --- a/agni/Makefile +++ b/agni/Makefile @@ -5,15 +5,13 @@ DOCKER_REPO ?= $(DOCKER_STAGE_REPO) DOCKER_IMAGE ?= agni DOCKER_TAG ?= master SBT_CMD = sbt -DDOCKER_REPO=$(DOCKER_REPO) -DDOCKER_TAG=${DOCKER_TAG} -SEARCH_SERVER ?= elasticsearch://10.240.0.18:9300 -PORT ?= 9000 autoformat-check: ../utils/scalafmt/scalafmt.sh --test run: $(call header, Run locally) - sbt api/run + sbt '~api/re-start' clean: $(call header, Cleaning) diff --git a/agni/core/resources/reference.conf b/agni/core/resources/reference.conf index 8b16a20874..c9bebdad37 100644 --- a/agni/core/resources/reference.conf +++ b/agni/core/resources/reference.conf @@ -6,6 +6,7 @@ app { http { interface = "0.0.0.0" + port = 9000 port = ${?PORT} } } diff --git a/agni/project/sbt-resolver.sbt b/agni/project/sbt-resolver.sbt new file mode 100644 index 0000000000..8682fad3db --- /dev/null +++ b/agni/project/sbt-resolver.sbt @@ -0,0 +1 @@ +addSbtPlugin("io.spray" % "sbt-revolver" % "0.8.0") From 38fab80d2ff3bca8cf12c6aa4ae4fdf75fc88471 Mon Sep 17 00:00:00 2001 From: Krzysztof Janosz Date: Wed, 28 Jun 2017 17:51:27 +0200 Subject: [PATCH 49/61] Fix some query functions Fix bool and raw query functions translation. Change boost to be set per function type instead of a context. --- .../core/app/foxcomm/agni/SearchService.scala | 2 +- agni/core/app/foxcomm/agni/dsl/query.scala | 116 +++++++++++------- .../interpreter/es/ESQueryInterpreter.scala | 70 ++++++----- agni/core/app/foxcomm/agni/package.scala | 22 +++- .../test/foxcomm/agni/dsl/QueryDslSpec.scala | 23 ++-- agni/core/test/resources/query/multiple.json | 15 ++- 6 files changed, 165 insertions(+), 83 deletions(-) diff --git a/agni/core/app/foxcomm/agni/SearchService.scala b/agni/core/app/foxcomm/agni/SearchService.scala index 105a608650..ad017d659b 100644 --- a/agni/core/app/foxcomm/agni/SearchService.scala +++ b/agni/core/app/foxcomm/agni/SearchService.scala @@ -55,7 +55,7 @@ class SearchService private (client: Client, qi: ESQueryInterpreter) { def evalQuery(builder: SearchRequestBuilder): Coeval[SearchRequestBuilder] = searchPayload match { case SearchPayload.es(query, _) ⇒ - Coeval.eval(builder.setQuery(Json.fromJsonObject(query).dump)) + Coeval.eval(builder.setQuery(Json.fromJsonObject(query).toBytes)) case SearchPayload.fc(query, _) ⇒ qi(query).map(builder.setQuery) } diff --git a/agni/core/app/foxcomm/agni/dsl/query.scala b/agni/core/app/foxcomm/agni/dsl/query.scala index 5e362d915b..4bc9dbea96 100644 --- a/agni/core/app/foxcomm/agni/dsl/query.scala +++ b/agni/core/app/foxcomm/agni/dsl/query.scala @@ -3,13 +3,11 @@ package foxcomm.agni.dsl import scala.language.higherKinds import cats.data.{NonEmptyList, NonEmptyVector} import cats.implicits._ -import cats.instances.either._ import io.circe._ import io.circe.generic.extras.semiauto._ import scala.util.Try import shapeless._ -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) object query { type QueryValueF[F[_], T] = T :+: F[T] :+: CNil type QueryValue[T] = QueryValueF[NonEmptyList, T] @@ -48,24 +46,13 @@ object query { Decoder[RangeBound[String]].map(Coproduct[RangeValue](_)) object Boostable { - private[this] val explicitBoostableRegex = "^(\\w+)\\^([0-9]*\\.?[0-9]+)$".r - private[this] val boostableRegex = "^(\\w+)\\^$".r + private[this] val boostableRegex = "^(\\w+)\\^([0-9]*\\.?[0-9]+)$".r - object explicit { - def unapply(s: String): Option[(String, Float)] = s match { - case explicitBoostableRegex(f, b) ⇒ Try(f → b.toFloat).toOption - case _ ⇒ None - } + def unapply(s: String): Option[(String, Float)] = s match { + case boostableRegex(f, b) ⇒ Try(f → b.toFloat).toOption + case _ ⇒ None } - private[this] val explicitMatcher = (explicit.unapply _).andThen(_.map { case (f, b) ⇒ f → Some(b) }) - - def unapply(s: String): Option[(String, Option[Float])] = - explicitMatcher(s) orElse (s match { - case boostableRegex(f) ⇒ Some(f → Some(default)) - case f ⇒ Some(f → None) - }) - def default: Float = 1.0f } @@ -98,18 +85,12 @@ object query { sealed trait QueryContext object QueryContext { - final case class must(boost: Option[Float]) extends QueryContext - final case class should(boost: Option[Float]) extends QueryContext - final case class not(boost: Option[Float]) extends QueryContext - - implicit val decodeQueryContext: Decoder[QueryContext] = Decoder.instance[QueryContext](c ⇒ - c.as[String].flatMap { - case Boostable("must", b) ⇒ Either.right(must(b)) - case Boostable("should", b) ⇒ Either.right(should(b)) - case Boostable.explicit("not", b) ⇒ Either.right(not(Some(b))) - case "not" ⇒ Either.right(not(None)) - case _ ⇒ Either.left(DecodingFailure("Invalid query context", c.history)) - }) + case object filter extends QueryContext + case object must extends QueryContext + case object should extends QueryContext + case object not extends QueryContext + + implicit val decodeQueryContext: Decoder[QueryContext] = deriveEnumerationDecoder[QueryContext] } sealed trait RangeFunction @@ -164,64 +145,110 @@ object query { sealed trait QueryFunction object QueryFunction { - sealed trait Basic extends QueryFunction + val Discriminator = "type" + + private def buildQueryFunctionDecoder[A <: QueryFunction](expectedTpe: String, decoder: Decoder[A])( + onBoost: (Float, HCursor, Decoder[A]) ⇒ Decoder.Result[A]) = + Decoder.instance { c ⇒ + val tpe = c.downField(Discriminator).focus.flatMap(_.asString) + tpe match { + case Some(Boostable(`expectedTpe`, b)) ⇒ onBoost(b, c, decoder) + case Some(`expectedTpe`) ⇒ decoder(c) + case _ ⇒ Either.left(DecodingFailure("Unknown query function type", c.history)) + } + } + + private def buildBoostableDecoder[A <: QueryFunction](expectedTpe: String)(decoder: Decoder[A]) = + buildQueryFunctionDecoder[A](expectedTpe, decoder)((boost, cursor, decoder) ⇒ + decoder.tryDecode(cursor.withFocus(_.mapObject(_.add("boost", Json.fromFloatOrNull(boost)))))) - sealed trait WithField extends Basic { this: QueryFunction ⇒ + private def buildDecoder[A <: QueryFunction](expectedTpe: String)(decoder: Decoder[A]): Decoder[A] = + buildQueryFunctionDecoder[A](expectedTpe, decoder)((_, cursor, _) ⇒ + Either.left(DecodingFailure(s"$expectedTpe query function is not boostable", cursor.history))) + + sealed trait WithField { this: QueryFunction ⇒ def field: QueryField + def boost: Option[Float] } - sealed trait WithContext extends Basic { this: QueryFunction ⇒ + sealed trait WithContext { this: QueryFunction ⇒ def ctx: QueryContext } - sealed trait TermLevel extends Basic with WithContext { this: QueryFunction ⇒ + sealed trait TermLevel extends WithContext { this: QueryFunction ⇒ def context: Option[QueryContext] - final def ctx: QueryContext = context.getOrElse(QueryContext.must(None)) + final def ctx: QueryContext = context.getOrElse(QueryContext.filter) } - sealed trait FullText extends Basic with WithContext with WithField { this: QueryFunction ⇒ + sealed trait FullText extends WithContext with WithField { this: QueryFunction ⇒ def context: Option[QueryContext] def in: Option[QueryField] - final def ctx: QueryContext = context.getOrElse(QueryContext.must(Some(Boostable.default))) + final def ctx: QueryContext = context.getOrElse(QueryContext.must) final def field: QueryField = in.getOrElse(QueryField.Single(Coproduct("_all"))) } final case class matches private (in: Option[QueryField], value: QueryValue[String], - context: Option[QueryContext]) + context: Option[QueryContext], + boost: Option[Float]) extends QueryFunction with FullText + object matches { + implicit val decodeMatches: Decoder[matches] = buildBoostableDecoder("matches")(deriveDecoder[matches]) + } - final case class equals private (in: QueryField, value: CompoundValue, context: Option[QueryContext]) + final case class equals private (in: QueryField, + value: CompoundValue, + context: Option[QueryContext], + boost: Option[Float]) extends QueryFunction with TermLevel with WithField { def field: QueryField = in } + object equals { + implicit val decodeEquals: Decoder[equals] = buildBoostableDecoder("equals")(deriveDecoder[equals]) + } final case class exists private (value: QueryField, context: Option[QueryContext]) extends QueryFunction with TermLevel + object exists { + implicit val decodeExists: Decoder[exists] = buildDecoder("exists")(deriveDecoder[exists]) + } - final case class range private (in: QueryField.Single, value: RangeValue, context: Option[QueryContext]) + final case class range private (in: QueryField.Single, + value: RangeValue, + context: Option[QueryContext], + boost: Option[Float]) extends QueryFunction with TermLevel with WithField { def field: QueryField.Single = in } + object range { + implicit val decodeRange: Decoder[range] = buildBoostableDecoder("range")(deriveDecoder[range]) + } final case class raw private (value: JsonObject, context: QueryContext) extends QueryFunction with WithContext { def ctx: QueryContext = context } + object raw { + implicit val decodeRaw: Decoder[raw] = buildDecoder("raw")(deriveDecoder[raw]) + } - final case class bool private (value: QueryValue[QueryFunction]) extends QueryFunction + final case class bool private (value: QueryValue[QueryFunction], context: QueryContext) + extends QueryFunction + with WithContext { + def ctx: QueryContext = context + } object bool { // TODO: make it configurable (?) val MaxDepth = 25 - implicit val decodeBool: Decoder[bool] = { + implicit val decodeBool: Decoder[bool] = buildDecoder[bool]("bool") { val decoder = deriveDecoder[bool] val depthField = "_depth" @@ -244,7 +271,12 @@ object query { } implicit val decodeQueryFunction: Decoder[QueryFunction] = - deriveDecoder[QueryFunction.Basic].map(identity) or Decoder[bool].map(identity) + Decoder[matches].map(identity[QueryFunction](_)) or + Decoder[equals].map(identity[QueryFunction](_)) or + Decoder[exists].map(identity[QueryFunction](_)) or + Decoder[range].map(identity[QueryFunction](_)) or + Decoder[raw].map(identity[QueryFunction](_)) or + Decoder[bool].map(identity[QueryFunction](_)) } final case class FCQuery(query: Option[NonEmptyList[QueryFunction]]) diff --git a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala index 71b6412a4d..d8f850cc2d 100644 --- a/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala +++ b/agni/core/app/foxcomm/agni/interpreter/es/ESQueryInterpreter.scala @@ -9,7 +9,7 @@ import foxcomm.agni.interpreter.QueryInterpreter import io.circe.JsonObject import org.elasticsearch.common.xcontent.{ToXContent, XContentBuilder} import org.elasticsearch.index.query._ -import shapeless.Coproduct +import scala.annotation.tailrec @SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.TraversableOps")) private[es] object ESQueryInterpreter @@ -18,39 +18,50 @@ private[es] object ESQueryInterpreter val State: (BoolQueryBuilder ⇒ Id[BoolQueryBuilder]) ⇒ State = Kleisli[Id, BoolQueryBuilder, BoolQueryBuilder] + private def buildNestedQuery(path: NonEmptyVector[String])(q: String ⇒ QueryBuilder): QueryBuilder = { + @tailrec def rec(path: Vector[String], acc: QueryBuilder): QueryBuilder = path match { + case Vector() :+ l ⇒ + QueryBuilders.nestedQuery(l, acc) + case i :+ l ⇒ + rec(i, QueryBuilders.nestedQuery(l, acc)) + } + + val combined = path.tail.iterator.scanLeft(path.head)((p, f) ⇒ s"$p.$f").toVector + rec(combined.init, q(combined.last)) + } + private implicit class RichBoolQueryBuilder(val b: BoolQueryBuilder) extends AnyVal { - def inContext(qf: QueryFunction.WithContext)(qb: QueryBuilder): Unit = qf.ctx match { - case QueryContext.must(None) ⇒ b.filter(qb) - case QueryContext.must(Some(boost)) ⇒ b.must(qb).boost(boost) - case QueryContext.should(boost) ⇒ b.should(qb).boost(boost.getOrElse(Boostable.default)) - case QueryContext.not(None) ⇒ b.mustNot(qb) - case QueryContext.not(Some(boost)) ⇒ - val boosting = QueryBuilders - .boostingQuery() - .negativeBoost(boost) - .negative(qb) - .boost(Boostable.default) - .positive(QueryBuilders.matchAllQuery()) - b.must(boosting) + def inContext(qf: QueryFunction.WithContext)(qb: QueryBuilder): BoolQueryBuilder = qf.ctx match { + case QueryContext.filter ⇒ b.filter(qb) + case QueryContext.must ⇒ b.must(qb) + case QueryContext.should ⇒ b.should(qb) + case QueryContext.not ⇒ b.mustNot(qb) } } + private implicit class RichBoostableQueryBuilder[B <: QueryBuilder with BoostableQueryBuilder[B]](val b: B) + extends AnyVal { + def withBoost(qf: QueryFunction.WithField): B = qf.boost.fold(b)(b.boost) + } + private implicit class RichField(val f: Field) extends AnyVal { def nest(q: String ⇒ QueryBuilder): QueryBuilder = - f.eliminate(q, - _.eliminate(fs ⇒ fs.toVector.init.foldRight(q(fs.toVector.last))(QueryBuilders.nestedQuery), - _.impossible)) + f.eliminate( + q, + _.eliminate( + buildNestedQuery(_)(q), + _.impossible + ) + ) } private final case class RawQueryBuilder(content: JsonObject) extends QueryBuilder { - def doXContent(builder: XContentBuilder, params: ToXContent.Params): Unit = { - builder.startObject() + + def doXContent(builder: XContentBuilder, params: ToXContent.Params): Unit = content.toMap.foreach { case (n, v) ⇒ - builder.rawField(n, v.dump) + builder.rawField(n, v.toSmile) } - builder.endObject() - } } def apply(qfs: NonEmptyList[QueryFunction]): State = State { b ⇒ @@ -61,15 +72,15 @@ private[es] object ESQueryInterpreter val inContext = b.inContext(qf) _ for (v ← qf.value.toList) { qf.field match { - case QueryField.Single(n) ⇒ inContext(n.nest(QueryBuilders.matchQuery(_, v))) + case QueryField.Single(n) ⇒ inContext(n.nest(QueryBuilders.matchQuery(_, v).withBoost(qf))) case QueryField.Multiple(ns) ⇒ val (sfs, nfs) = ns.foldLeft(Vector.empty[String] → Vector.empty[NonEmptyVector[String]]) { case ((sAcc, nAcc), f) ⇒ f.select[String].fold(sAcc)(sAcc :+ _) → f.select[NonEmptyVector[String]].fold(nAcc)(nAcc :+ _) } - inContext(QueryBuilders.multiMatchQuery(v, sfs: _*)) - nfs.foreach(nf ⇒ inContext(Coproduct[Field](nf).nest(QueryBuilders.matchQuery(_, v)))) + if (sfs.nonEmpty) inContext(QueryBuilders.multiMatchQuery(v, sfs: _*).withBoost(qf)) + nfs.foreach(nf ⇒ inContext(buildNestedQuery(nf)(QueryBuilders.matchQuery(_, v).withBoost(qf)))) } } b @@ -81,8 +92,8 @@ private[es] object ESQueryInterpreter for (f ← qf.in.toList) { inContext { vs match { - case v :: Nil ⇒ f.nest(QueryBuilders.termQuery(_, v)) - case _ ⇒ f.nest(QueryBuilders.termsQuery(_, vs: _*)) + case v :: Nil ⇒ f.nest(QueryBuilders.termQuery(_, v).withBoost(qf)) + case _ ⇒ f.nest(QueryBuilders.termsQuery(_, vs: _*).withBoost(qf)) } } } @@ -98,7 +109,7 @@ private[es] object ESQueryInterpreter def rangeF(qf: QueryFunction.range): State = State { b ⇒ b.inContext(qf) { qf.in.field.nest { f ⇒ - val builder = QueryBuilders.rangeQuery(f) + val builder = QueryBuilders.rangeQuery(f).withBoost(qf) val value = qf.value.unify value.lower.foreach { case (RangeFunction.Gt, v) ⇒ builder.gt(v) @@ -120,6 +131,7 @@ private[es] object ESQueryInterpreter } def boolF(qf: QueryFunction.bool): State = State { b ⇒ - qf.value.toNEL.foldM(b)((b, qf) ⇒ eval(qf)(b): Id[BoolQueryBuilder]) + b.inContext(qf)(apply(qf.value.toNEL)(QueryBuilders.boolQuery())) + b } } diff --git a/agni/core/app/foxcomm/agni/package.scala b/agni/core/app/foxcomm/agni/package.scala index c28b5f629b..61c553e078 100644 --- a/agni/core/app/foxcomm/agni/package.scala +++ b/agni/core/app/foxcomm/agni/package.scala @@ -1,12 +1,18 @@ package foxcomm +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.dataformat.smile.SmileFactory import io.circe.generic.extras.Configuration import io.circe.{Json, Printer} +import java.io.ByteArrayOutputStream import monix.eval.Task import org.elasticsearch.action.ActionListener import scala.concurrent.Promise package object agni { + private[this] val smileFactory = new SmileFactory() + private[this] val jsonFactory = new JsonFactory() + implicit val configuration: Configuration = Configuration.default.withDefaults.withDiscriminator("type").withSnakeCaseKeys @@ -21,7 +27,21 @@ package object agni { p.future } + @SuppressWarnings(Array("org.wartremover.warts.While")) implicit class RichJson(val j: Json) extends AnyVal { - def dump: Array[Byte] = Printer.noSpaces.prettyByteBuffer(j).array() + def toBytes: Array[Byte] = Printer.noSpaces.prettyByteBuffer(j).array() + + def toSmile: Array[Byte] = { + val bos = new ByteArrayOutputStream() + val jg = smileFactory.createGenerator(bos) + val jp = jsonFactory.createParser(j.toBytes) + try while (jp.nextToken() ne null) { + jg.copyCurrentEvent(jp) + } finally { + jp.close() + jg.close() + } + bos.toByteArray + } } } diff --git a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala index 94515af8f6..342807a3fb 100644 --- a/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala +++ b/agni/core/test/foxcomm/agni/dsl/QueryDslSpec.scala @@ -5,6 +5,7 @@ import foxcomm.agni.dsl.query._ import io.circe.{Json, JsonObject} import io.circe.parser._ import org.scalatest.EitherValues._ +import org.scalatest.OptionValues._ import org.scalatest.{Assertion, FlatSpec, Matchers} import scala.annotation.tailrec import scala.io.Source @@ -31,7 +32,7 @@ class QueryDslSpec extends FlatSpec with Matchers { json.as[FCQuery].right.value.query.map(_.toList).getOrElse(Nil) assertQueryFunction[QueryFunction.equals](queries.head) { equals ⇒ equals.field.toList should === (List(Coproduct[Field]("slug"))) - equals.ctx should === (QueryContext.must(Some(Boostable.default))) + equals.ctx should === (QueryContext.must) equals.context should be('defined) equals.value.toList should === (List("awesome", "whatever")) } @@ -40,13 +41,14 @@ class QueryDslSpec extends FlatSpec with Matchers { List(Coproduct[Field]("title"), Coproduct[Field]("description"), Coproduct[Field](NonEmptyVector.of("skus", "code")))) - matches.ctx should === (QueryContext.should(Some(0.5f))) + matches.ctx should === (QueryContext.should) + matches.boost.value should === (0.5f) matches.context should be('defined) matches.value.toList should === (List("food", "drink")) } assertQueryFunction[QueryFunction.range](queries(2)) { range ⇒ range.field.toList should === (List(Coproduct[Field]("price"))) - range.ctx should === (QueryContext.must(None)) + range.ctx should === (QueryContext.filter) range.context should be('empty) range.value.unify.toMap.mapValues(_.toString) should === ( Map( @@ -56,16 +58,22 @@ class QueryDslSpec extends FlatSpec with Matchers { } assertQueryFunction[QueryFunction.exists](queries(3)) { exists ⇒ exists.value.toList should === (List(Coproduct[Field]("archivedAt"))) - exists.ctx should === (QueryContext.not(None)) + exists.ctx should === (QueryContext.not) exists.context should be('defined) } - assertQueryFunction[QueryFunction.bool](queries(4)) { bool ⇒ + assertQueryFunction[QueryFunction.raw](queries(4)) { raw ⇒ + raw.context should === (QueryContext.filter) + raw.value should === (JsonObject.singleton("match_all", Json.fromJsonObject(JsonObject.empty))) + } + assertQueryFunction[QueryFunction.bool](queries(5)) { bool ⇒ + bool.context should === (QueryContext.should) val qfs = bool.value.toNEL assertQueryFunction[QueryFunction.equals](qfs.head) { equals ⇒ equals.field.toList should === (List(Coproduct[Field]("context"))) equals.value.toList should === (List("default")) } assertQueryFunction[QueryFunction.bool](qfs.tail.head) { bool ⇒ + bool.context should === (QueryContext.not) assertQueryFunction[QueryFunction.exists](bool.value.toNEL.head) { exists ⇒ exists.value.toList should === (List(Coproduct[Field]("context"))) } @@ -86,8 +94,9 @@ class QueryDslSpec extends FlatSpec with Matchers { deepBool(boolDepth - 1, JsonObject.fromMap( Map( - "type" → Json.fromString("bool"), - "value" → Json.fromJsonObject(embed) + "type" → Json.fromString("bool"), + "context" → Json.fromString("filter"), + "value" → Json.fromJsonObject(embed) ))) else Json.fromJsonObject(embed) diff --git a/agni/core/test/resources/query/multiple.json b/agni/core/test/resources/query/multiple.json index fe009a03b5..20a4543aa3 100644 --- a/agni/core/test/resources/query/multiple.json +++ b/agni/core/test/resources/query/multiple.json @@ -1,13 +1,13 @@ [ { "type": "equals", - "context": "must^", + "context": "must", "in": "slug", "value": [ "awesome", "whatever" ] }, { - "type": "matches", - "context": "should^.5", + "type": "matches^.5", + "context": "should", "in": [ "title", "description", "skus.code" ], "value": [ "food", "drink" ] }, @@ -24,8 +24,16 @@ "context": "not", "value": "archivedAt" }, + { + "type": "raw", + "context": "filter", + "value": { + "match_all": {} + } + }, { "type": "bool", + "context": "should", "value": [ { "type": "equals", @@ -34,6 +42,7 @@ }, { "type": "bool", + "context": "not", "value": { "type": "exists", "value": "context" From b3d2d2ce54493842666e90a5739164b08f8067cc Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 13:55:13 +0300 Subject: [PATCH 50/61] Revert the changes for Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 703cbc9864..6a5d871704 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ config: up: $(call header, Creating Appliance) - ansible-playbook -vvvv --user=$(GOOGLE_SSH_USERNAME) --private-key=$(GOOGLE_SSH_KEY) --extra-vars '{"FIRST_RUN": true}' tabernacle/ansible/goldrush_appliance.yml + ansible-playbook --user=$(GOOGLE_SSH_USERNAME) --private-key=$(GOOGLE_SSH_KEY) --extra-vars '{"FIRST_RUN": true}' tabernacle/ansible/goldrush_appliance.yml @cat goldrush.log destroy: From 534bd883989f215c651cd0bc72c01446b9203c8b Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 13:57:31 +0300 Subject: [PATCH 51/61] Change cpu and mem --- .../dev/marathon_groups/templates/core-backend/agni.json.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index 55d148f44d..603eb2ab72 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -1,8 +1,8 @@ { "id": "agni", "cmd": null, - "cpus": 0.5, - "mem": 256, + "cpus": 1, + "mem": 640, "disk": 0, "instances": 1, "constraints": [], From 5a78c930b8667c432b66271e5e9b9e3fe11dcc12 Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 13:58:18 +0300 Subject: [PATCH 52/61] Set JAVA_OPTS --- .../dev/marathon_groups/templates/core-backend/agni.json.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index 603eb2ab72..bacd36d8da 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -20,7 +20,7 @@ "env": { "PORT": "{{agni_port}}", "SEARCH_SERVER": "elasticsearch://{{search_server}}", - "JAVA_OPTS":"" + "JAVA_OPTS":"-XX:+UseConcMarkSweepGC -Xms512m -Xmx512m" }, "ports": [{{agni_port}}], "healthChecks": [{ From 740f45d03536e17baf163b3dba78f7c30345d82e Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 15:48:32 +0300 Subject: [PATCH 53/61] Assign a random port to agni --- tabernacle/ansible/roles/dev/balancer/templates/services.j2 | 3 +-- .../dev/marathon_groups/templates/core-backend/agni.json.j2 | 1 - tabernacle/ansible/roles/dev/nginx/templates/services.j2 | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 index b2395c31a6..2612f3c140 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 @@ -58,9 +58,8 @@ upstream sol { << else >> server {{solomon_server}} fail_timeout=30s max_fails=10; << end >> } -# We use port 9000 for agni upstream agni { - << range service "agni" >> server << .Address >>:9000 max_fails=10 fail_timeout=30s weight=1; + << range service "agni" >> server << .Address >>.Port max_fails=10 fail_timeout=30s weight=1; << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> } diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index bacd36d8da..6c7d638e2b 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -18,7 +18,6 @@ "maximumOverCapacity": 0 }, "env": { - "PORT": "{{agni_port}}", "SEARCH_SERVER": "elasticsearch://{{search_server}}", "JAVA_OPTS":"-XX:+UseConcMarkSweepGC -Xms512m -Xmx512m" }, diff --git a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 index 6f719b1977..d448e4ed5c 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 @@ -68,9 +68,8 @@ upstream geronimo { << else >> server {{geronimo_server}} fail_timeout=30s max_fails=10; << end >> } -# We use port 9000 for agni upstream agni { - << range service "agni" >> server << .Address >>:9000 max_fails=10 fail_timeout=30s weight=1; + << range service "agni" >> server << .Address >>.Port max_fails=10 fail_timeout=30s weight=1; << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> } From adaa6168c362934f8e437e85d075bda6812ac554 Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 16:25:27 +0300 Subject: [PATCH 54/61] Remove port from healthChecks --- .../dev/marathon_groups/templates/core-backend/agni.json.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index 6c7d638e2b..a777453382 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -29,8 +29,7 @@ "intervalSeconds": 30, "timeoutSeconds": 20, "maxConsecutiveFailures": 3, - "ignoreHttp1xx": false, - "port": {{agni_port}} + "ignoreHttp1xx": false }], "container": { "type": "DOCKER", From 6c467f307a082ab11bed67f2effb4c9cd2460a5f Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 16:34:05 +0300 Subject: [PATCH 55/61] Remove another ports field --- .../dev/marathon_groups/templates/core-backend/agni.json.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 index a777453382..0537da75b5 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-backend/agni.json.j2 @@ -21,7 +21,6 @@ "SEARCH_SERVER": "elasticsearch://{{search_server}}", "JAVA_OPTS":"-XX:+UseConcMarkSweepGC -Xms512m -Xmx512m" }, - "ports": [{{agni_port}}], "healthChecks": [{ "path": "/ping", "protocol": "HTTP", From e83bddd1ba5f488927c7268a5a96ea45d922410b Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 16:39:10 +0300 Subject: [PATCH 56/61] Fix the typo --- tabernacle/ansible/roles/dev/balancer/templates/services.j2 | 2 +- tabernacle/ansible/roles/dev/nginx/templates/services.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 index 2612f3c140..7ca59858bd 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 @@ -59,7 +59,7 @@ upstream sol { } upstream agni { - << range service "agni" >> server << .Address >>.Port max_fails=10 fail_timeout=30s weight=1; + << range service "agni" >> server << .Address >>:<< .Port max_fails=10 fail_timeout=30s weight=1; << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> } diff --git a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 index d448e4ed5c..4d9709abd6 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 @@ -69,7 +69,7 @@ upstream geronimo { } upstream agni { - << range service "agni" >> server << .Address >>.Port max_fails=10 fail_timeout=30s weight=1; + << range service "agni" >> server << .Address >>:<< .Port max_fails=10 fail_timeout=30s weight=1; << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> } From 976241c9da76e3e373679478a38130a4e4f69623 Mon Sep 17 00:00:00 2001 From: "Eugene Sypachev (@Axblade)" Date: Fri, 30 Jun 2017 16:39:36 +0300 Subject: [PATCH 57/61] wire tables to agni with raw queries --- ashes/src/lib/agni.js | 21 +++++++++++++++++++ ashes/src/lib/search.js | 11 ++++++---- ashes/src/modules/carts/list.js | 2 +- ashes/src/modules/catalog/list.js | 2 +- ashes/src/modules/coupons/coupon-codes.js | 3 +-- ashes/src/modules/coupons/list.js | 3 +-- .../customer-groups/details/customers-list.js | 2 +- ashes/src/modules/customer-groups/list.js | 2 +- ashes/src/modules/customers/items.js | 2 +- ashes/src/modules/customers/list.js | 2 +- .../store-credit-transactions/transactions.js | 3 +-- ashes/src/modules/customers/store-credits.js | 3 +-- .../customers/transactions/transactions.js | 2 +- ashes/src/modules/gift-cards/list.js | 2 +- .../gift-cards/transactions/transactions.js | 3 +-- ashes/src/modules/inventory/list.js | 2 +- .../inventory/transactions/transactions.js | 2 +- .../src/modules/live-search/searches-data.js | 2 +- ashes/src/modules/notes.js | 2 +- ashes/src/modules/orders/list.js | 2 +- ashes/src/modules/orders/new-order.js | 2 +- ashes/src/modules/products/list.js | 2 +- ashes/src/modules/promotions/list.js | 2 +- ashes/src/modules/skus/list.js | 2 +- ashes/src/modules/taxonomies/list.js | 2 +- .../modules/taxons/details/products-list.js | 2 +- ashes/src/modules/taxons/list.js | 2 +- ashes/src/modules/users/list.js | 2 +- 28 files changed, 54 insertions(+), 35 deletions(-) create mode 100644 ashes/src/lib/agni.js diff --git a/ashes/src/lib/agni.js b/ashes/src/lib/agni.js new file mode 100644 index 0000000000..f16333bef5 --- /dev/null +++ b/ashes/src/lib/agni.js @@ -0,0 +1,21 @@ +/* @flow */ + +import { request as baseRequest } from './api'; + +type TArgs = [string, Object]; + +function searchURI(uri: string): string { + return `/api/advanced-search/admin/${uri}`; +} + +function request(method: string, uri: string, data: Object): Promise<*> { + const payload = { + "type": "es", + "query": {...data}, + }; + return baseRequest(method, searchURI(uri), payload); +} + +export function post(...args: TArgs): Promise<*> { + return request('POST', ...args); +} diff --git a/ashes/src/lib/search.js b/ashes/src/lib/search.js index 2c796059c0..207c5e3476 100644 --- a/ashes/src/lib/search.js +++ b/ashes/src/lib/search.js @@ -1,18 +1,21 @@ +/* @flow */ import { request as baseRequest } from './api'; -function searchURI(uri) { +type TArgs = [string, Object]; + +function searchURI(uri: string): string { return `/api/search/admin/${uri}`; } -function request(method, uri, data) { +function request(method: string, uri: string, data: Object): Promise<*> { return baseRequest(method, searchURI(uri), data); } -export function get(...args) { +export function get(...args: TArgs): Promise<*> { return request('GET', ...args); } -export function post(...args) { +export function post(...args: TArgs): Promise<*> { return request('POST', ...args); } diff --git a/ashes/src/modules/carts/list.js b/ashes/src/modules/carts/list.js index 6206d4a7ad..55204eb14d 100644 --- a/ashes/src/modules/carts/list.js +++ b/ashes/src/modules/carts/list.js @@ -6,7 +6,7 @@ import { addNativeFilters, addShouldFilters } from '../../elastic/common'; const { reducer, actions } = makeLiveSearch( 'carts.list', searchTerms, - 'carts_search_view/_search', + 'carts_search_view', 'cartsScope', { processQuery: (query) => { diff --git a/ashes/src/modules/catalog/list.js b/ashes/src/modules/catalog/list.js index 44703a9e9b..7540594e5b 100644 --- a/ashes/src/modules/catalog/list.js +++ b/ashes/src/modules/catalog/list.js @@ -6,7 +6,7 @@ import searchTerms from './search-terms'; const { reducer, actions } = makeLiveSearch( 'catalogs.list', searchTerms, - 'catalogs_search_view/_search', + 'catalogs_search_view', 'catalogsScope', { rawSorts: ['name'], diff --git a/ashes/src/modules/coupons/coupon-codes.js b/ashes/src/modules/coupons/coupon-codes.js index e244da9840..e87c251bb4 100644 --- a/ashes/src/modules/coupons/coupon-codes.js +++ b/ashes/src/modules/coupons/coupon-codes.js @@ -1,4 +1,3 @@ - import makeLiveSearch from '../live-search'; const searchTerms = [ @@ -27,7 +26,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'coupons.couponCodes', searchTerms, - 'coupon_codes_search_view/_search', + 'coupon_codes_search_view', 'couponCodesScope', { skipInitialFetch: true }, diff --git a/ashes/src/modules/coupons/list.js b/ashes/src/modules/coupons/list.js index c0003a6dc1..5ee96e6f4b 100644 --- a/ashes/src/modules/coupons/list.js +++ b/ashes/src/modules/coupons/list.js @@ -1,4 +1,3 @@ - import makeLiveSearch from '../live-search'; const searchTerms = [ @@ -26,7 +25,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'coupons.list', searchTerms, - 'coupons_search_view/_search', + 'coupons_search_view', 'couponsScope', { initialState: { sortBy: '-createdAt' } diff --git a/ashes/src/modules/customer-groups/details/customers-list.js b/ashes/src/modules/customer-groups/details/customers-list.js index a20268308e..86d489aeea 100644 --- a/ashes/src/modules/customer-groups/details/customers-list.js +++ b/ashes/src/modules/customer-groups/details/customers-list.js @@ -21,7 +21,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'customerGroups.details.customers', searchTerms, - 'customers_search_view/_search', + 'customers_search_view', 'customersScope', { skipInitialFetch: true, diff --git a/ashes/src/modules/customer-groups/list.js b/ashes/src/modules/customer-groups/list.js index 679b48e920..3ed230d0d5 100644 --- a/ashes/src/modules/customer-groups/list.js +++ b/ashes/src/modules/customer-groups/list.js @@ -5,7 +5,7 @@ import makeLiveSearch from '../live-search'; const { reducer, actions } = makeLiveSearch( 'customerGroups.list', [], - 'customer_groups_search_view/_search', + 'customer_groups_search_view', 'customerGroupsScope', { processQuery: (query) => addNativeFilters(query,[dsl.existsFilter('deletedAt', 'missing')]), diff --git a/ashes/src/modules/customers/items.js b/ashes/src/modules/customers/items.js index d72ff02aee..8c8dd5ff9e 100644 --- a/ashes/src/modules/customers/items.js +++ b/ashes/src/modules/customers/items.js @@ -41,7 +41,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'customers.items', searchTerms, - 'customer_items_view/_search', + 'customer_items_view', 'customerItemsScope', { skipInitialFetch: true, diff --git a/ashes/src/modules/customers/list.js b/ashes/src/modules/customers/list.js index acb5314b9a..b46176f8d4 100644 --- a/ashes/src/modules/customers/list.js +++ b/ashes/src/modules/customers/list.js @@ -6,7 +6,7 @@ import * as dsl from 'elastic/dsl'; const { reducer, actions } = makeLiveSearch( 'customers.list', searchTerms, - 'customers_search_view/_search', + 'customers_search_view', 'customersScope', { initialState: { sortBy: '-joinedAt' }, diff --git a/ashes/src/modules/customers/store-credit-transactions/transactions.js b/ashes/src/modules/customers/store-credit-transactions/transactions.js index 53b2af3af8..01c246e174 100644 --- a/ashes/src/modules/customers/store-credit-transactions/transactions.js +++ b/ashes/src/modules/customers/store-credit-transactions/transactions.js @@ -1,4 +1,3 @@ - import makeLiveSearch from 'modules/live-search'; const searchTerms = [ @@ -81,7 +80,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'customers.storeCreditTransactions.list', searchTerms, - 'store_credit_transactions_search_view/_search', + 'store_credit_transactions_search_view', 'customerStoreCreditTransactionsScope', { initialState: { sortBy: '-createdAt' }, skipInitialFetch: true diff --git a/ashes/src/modules/customers/store-credits.js b/ashes/src/modules/customers/store-credits.js index e7f9e9770e..a715f89d7c 100644 --- a/ashes/src/modules/customers/store-credits.js +++ b/ashes/src/modules/customers/store-credits.js @@ -1,4 +1,3 @@ - import makeLiveSearch from '../live-search'; const searchTerms = [ @@ -65,7 +64,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'customers.storeCredits', searchTerms, - 'store_credits_search_view/_search', + 'store_credits_search_view', 'customerStoreCreditsScope', { skipInitialFetch: true } diff --git a/ashes/src/modules/customers/transactions/transactions.js b/ashes/src/modules/customers/transactions/transactions.js index da25882331..9fbe923e99 100644 --- a/ashes/src/modules/customers/transactions/transactions.js +++ b/ashes/src/modules/customers/transactions/transactions.js @@ -100,7 +100,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'customers.transactions.list', searchTerms, - 'orders_search_view/_search', + 'orders_search_view', 'customerTransactionsScope', { initialState: { sortBy: '-placedAt' }, skipInitialFetch: true diff --git a/ashes/src/modules/gift-cards/list.js b/ashes/src/modules/gift-cards/list.js index f56bcb9f73..849d8d647b 100644 --- a/ashes/src/modules/gift-cards/list.js +++ b/ashes/src/modules/gift-cards/list.js @@ -7,7 +7,7 @@ import searchTerms from './search-terms'; const { reducer, actions } = makeLiveSearch( 'giftCards.list', searchTerms, - 'gift_cards_search_view/_search', + 'gift_cards_search_view', 'giftCardsScope', { initialState: { sortBy: '-createdAt' } diff --git a/ashes/src/modules/gift-cards/transactions/transactions.js b/ashes/src/modules/gift-cards/transactions/transactions.js index 31b5b2f50f..fcf2192dd3 100644 --- a/ashes/src/modules/gift-cards/transactions/transactions.js +++ b/ashes/src/modules/gift-cards/transactions/transactions.js @@ -1,4 +1,3 @@ - import makeLiveSearch from 'modules/live-search'; const searchTerms = [ @@ -59,7 +58,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'giftCards.transactions.list', searchTerms, - 'gift_card_transactions_view/_search', + 'gift_card_transactions_view', 'giftCardTransactionsScope', { skipInitialFetch: true } diff --git a/ashes/src/modules/inventory/list.js b/ashes/src/modules/inventory/list.js index b349c27ed0..9fdb21b8f5 100644 --- a/ashes/src/modules/inventory/list.js +++ b/ashes/src/modules/inventory/list.js @@ -55,7 +55,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'inventory.list', searchTerms, - 'inventory_search_view/_search', + 'inventory_search_view', 'inventoryScope', { initialState: { sortBy: '-createdAt' }, diff --git a/ashes/src/modules/inventory/transactions/transactions.js b/ashes/src/modules/inventory/transactions/transactions.js index a73cab3bd9..1692079fb4 100644 --- a/ashes/src/modules/inventory/transactions/transactions.js +++ b/ashes/src/modules/inventory/transactions/transactions.js @@ -51,7 +51,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'inventory.transactions.list', searchTerms, - 'inventory_transactions_search_view/_search', + 'inventory_transactions_search_view', 'inventoryScope', { initialState: { sortBy: '-createdAt' }, diff --git a/ashes/src/modules/live-search/searches-data.js b/ashes/src/modules/live-search/searches-data.js index c2c4c1af59..e0e1971e47 100644 --- a/ashes/src/modules/live-search/searches-data.js +++ b/ashes/src/modules/live-search/searches-data.js @@ -1,6 +1,6 @@ import _ from 'lodash'; import { dissoc } from 'sprout-data'; -import { post } from '../../lib/search'; +import { post } from '../../lib/agni'; import { createReducer } from 'redux-act'; import { createNsAction } from './../utils'; import { toQuery, addNativeFilters } from '../../elastic/common'; diff --git a/ashes/src/modules/notes.js b/ashes/src/modules/notes.js index 1d68c786cc..0f3d85943a 100644 --- a/ashes/src/modules/notes.js +++ b/ashes/src/modules/notes.js @@ -13,7 +13,7 @@ function geCurrentEntity(state) { const {reducer, actions} = makeLiveSearch( 'notes.list', [], - 'notes_search_view/_search', + 'notes_search_view', null, { processQuery: (query, {getState}) => { const currentEntity = geCurrentEntity(getState()); diff --git a/ashes/src/modules/orders/list.js b/ashes/src/modules/orders/list.js index 1e2f91d2a5..74b857c268 100644 --- a/ashes/src/modules/orders/list.js +++ b/ashes/src/modules/orders/list.js @@ -4,7 +4,7 @@ import searchTerms from './search-terms'; const { reducer, actions } = makeLiveSearch( 'orders.list', searchTerms, - 'orders_search_view/_search', + 'orders_search_view', 'ordersScope', { initialState: { sortBy: '-placedAt' }, diff --git a/ashes/src/modules/orders/new-order.js b/ashes/src/modules/orders/new-order.js index 2e283cdb38..bd5186d3f8 100644 --- a/ashes/src/modules/orders/new-order.js +++ b/ashes/src/modules/orders/new-order.js @@ -9,7 +9,7 @@ const emptyFilters = []; const emptyPhrase = ''; const quickSearch = makeQuickSearch( 'orders.newOrder.customers', - 'customers_search_view/_search', + 'customers_search_view', emptyFilters, emptyPhrase ); diff --git a/ashes/src/modules/products/list.js b/ashes/src/modules/products/list.js index db4138801a..3dc5eef8e0 100644 --- a/ashes/src/modules/products/list.js +++ b/ashes/src/modules/products/list.js @@ -9,7 +9,7 @@ import searchTerms from './search-terms'; const { reducer, actions } = makeLiveSearch( 'products.list', searchTerms, - 'products_search_view/_search', + 'products_search_view', 'productsScope', { rawSorts: ['title'], diff --git a/ashes/src/modules/promotions/list.js b/ashes/src/modules/promotions/list.js index 35c11a4701..b53850ef21 100644 --- a/ashes/src/modules/promotions/list.js +++ b/ashes/src/modules/promotions/list.js @@ -44,7 +44,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'promotions.list', searchTerms, - 'promotions_search_view/_search', + 'promotions_search_view', 'promotionsScope', { initialState: { sortBy: '-createdAt' } diff --git a/ashes/src/modules/skus/list.js b/ashes/src/modules/skus/list.js index 59921b1cd8..f345d79ab2 100644 --- a/ashes/src/modules/skus/list.js +++ b/ashes/src/modules/skus/list.js @@ -39,7 +39,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'skus.list', searchTerms, - 'sku_search_view/_search', + 'sku_search_view', 'skusScope', { rawSorts: ['title'], diff --git a/ashes/src/modules/taxonomies/list.js b/ashes/src/modules/taxonomies/list.js index 1732ddccc5..8deedf81c3 100644 --- a/ashes/src/modules/taxonomies/list.js +++ b/ashes/src/modules/taxonomies/list.js @@ -6,7 +6,7 @@ import searchTerms from './search-terms'; const { reducer, actions } = makeLiveSearch( 'taxonomies.list', searchTerms, - 'taxonomies_search_view/_search', + 'taxonomies_search_view', 'taxonomiesScope', { initialState: { sortBy: 'name' }, diff --git a/ashes/src/modules/taxons/details/products-list.js b/ashes/src/modules/taxons/details/products-list.js index f947d7628f..c3cf0a2abb 100644 --- a/ashes/src/modules/taxons/details/products-list.js +++ b/ashes/src/modules/taxons/details/products-list.js @@ -6,7 +6,7 @@ import productsSearchTerms from 'modules/products/search-terms'; const { reducer, actions } = makeLiveSearch( 'taxons.details.products', productsSearchTerms, - 'products_search_view/_search', + 'products_search_view', 'productsScope', { initialState: { sortBy: 'name' }, diff --git a/ashes/src/modules/taxons/list.js b/ashes/src/modules/taxons/list.js index 8e9375ed4c..bacf8c3fb6 100644 --- a/ashes/src/modules/taxons/list.js +++ b/ashes/src/modules/taxons/list.js @@ -4,7 +4,7 @@ import makeLiveSearch from 'modules/live-search'; const searchTerms = []; const storeLocation = 'taxons.list'; -const searchView = 'taxons_search_view/_search'; +const searchView = 'taxons_search_view'; const scope = 'taxonsScope'; const { reducer, actions } = makeLiveSearch( diff --git a/ashes/src/modules/users/list.js b/ashes/src/modules/users/list.js index 0f175e985f..6d33243652 100644 --- a/ashes/src/modules/users/list.js +++ b/ashes/src/modules/users/list.js @@ -26,7 +26,7 @@ const searchTerms = [ const { reducer, actions } = makeLiveSearch( 'users.list', searchTerms, - 'store_admins_search_view/_search', + 'store_admins_search_view', 'storeAdminsScope', { initialState: { sortBy: '-createdAt' }, From 08eb32582605cf0bcd45f2f36d9a4a1c0c443dc5 Mon Sep 17 00:00:00 2001 From: "Eugene Sypachev (@Axblade)" Date: Fri, 30 Jun 2017 16:56:40 +0300 Subject: [PATCH 58/61] fix lint --- ashes/src/lib/agni.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ashes/src/lib/agni.js b/ashes/src/lib/agni.js index f16333bef5..0bb8cebe38 100644 --- a/ashes/src/lib/agni.js +++ b/ashes/src/lib/agni.js @@ -10,8 +10,8 @@ function searchURI(uri: string): string { function request(method: string, uri: string, data: Object): Promise<*> { const payload = { - "type": "es", - "query": {...data}, + type: 'es', + query: {...data}, }; return baseRequest(method, searchURI(uri), payload); } From c852c042b30bc0b0ce8e1894b0a95b975e478993 Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 30 Jun 2017 17:42:18 +0300 Subject: [PATCH 59/61] Another typo --- tabernacle/ansible/roles/dev/balancer/templates/services.j2 | 2 +- tabernacle/ansible/roles/dev/nginx/templates/services.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 index 7ca59858bd..d9a49a4613 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 @@ -59,7 +59,7 @@ upstream sol { } upstream agni { - << range service "agni" >> server << .Address >>:<< .Port max_fails=10 fail_timeout=30s weight=1; + << range service "agni" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> } diff --git a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 index 4d9709abd6..38e21debb4 100644 --- a/tabernacle/ansible/roles/dev/nginx/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/nginx/templates/services.j2 @@ -69,7 +69,7 @@ upstream geronimo { } upstream agni { - << range service "agni" >> server << .Address >>:<< .Port max_fails=10 fail_timeout=30s weight=1; + << range service "agni" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; << else >> server {{agni_server}} fail_timeout=30s max_fails=10; << end >> } From 8b70e61e97bf9f8e01d5191f0bff9e8d544cc0ff Mon Sep 17 00:00:00 2001 From: "Eugene Sypachev (@Axblade)" Date: Wed, 5 Jul 2017 18:55:48 +0300 Subject: [PATCH 60/61] use class instead of plain functions --- ashes/src/elastic/activities.js | 5 ++-- ashes/src/lib/agni.js | 28 +++++++++++-------- .../src/modules/live-search/searches-data.js | 4 +-- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/ashes/src/elastic/activities.js b/ashes/src/elastic/activities.js index d8243f0030..ecc6b41fb7 100644 --- a/ashes/src/elastic/activities.js +++ b/ashes/src/elastic/activities.js @@ -1,5 +1,6 @@ import _ from 'lodash'; -import { post } from '../lib/search'; +import Agni from 'lib/agni'; +import { post } from 'lib/search'; import moment from 'moment'; import * as dsl from './dsl'; @@ -58,7 +59,7 @@ export default function searchActivities(fromActivity = null, trailParams, days const q = queryFirstActivity(); - promise = post('scoped_activity_trails/_search', q) + promise = Agni.post('scoped_activity_trails', q) .then(response => { const result = _.isEmpty(response.result) ? [] : response.result; _.set(response, 'result', result); diff --git a/ashes/src/lib/agni.js b/ashes/src/lib/agni.js index 0bb8cebe38..5d6203274b 100644 --- a/ashes/src/lib/agni.js +++ b/ashes/src/lib/agni.js @@ -4,18 +4,22 @@ import { request as baseRequest } from './api'; type TArgs = [string, Object]; -function searchURI(uri: string): string { - return `/api/advanced-search/admin/${uri}`; -} +class Agni { + searchURI(uri: string): string { + return `/api/advanced-search/admin/${uri}`; + } -function request(method: string, uri: string, data: Object): Promise<*> { - const payload = { - type: 'es', - query: {...data}, - }; - return baseRequest(method, searchURI(uri), payload); -} + request(method: string, uri: string, data: Object): Promise<*> { + const payload = { + type: 'es', + query: {...data}, + }; + return baseRequest(method, this.searchURI(uri), payload); + } -export function post(...args: TArgs): Promise<*> { - return request('POST', ...args); + search(...args: TArgs): Promise<*> { + return this.request('POST', ...args); + } } + +export default new Agni(); diff --git a/ashes/src/modules/live-search/searches-data.js b/ashes/src/modules/live-search/searches-data.js index e0e1971e47..5223817c68 100644 --- a/ashes/src/modules/live-search/searches-data.js +++ b/ashes/src/modules/live-search/searches-data.js @@ -1,6 +1,6 @@ import _ from 'lodash'; import { dissoc } from 'sprout-data'; -import { post } from '../../lib/agni'; +import Agni from '../../lib/agni'; import { createReducer } from 'redux-act'; import { createNsAction } from './../utils'; import { toQuery, addNativeFilters } from '../../elastic/common'; @@ -59,7 +59,7 @@ export default function makeDataInSearches(namespace, esUrl, options = {}) { dispatch(saveRawQuery(jsonQuery)); - const promise = post(addPaginationParams(esUrl, searchState), processQuery(jsonQuery, { searchState, getState })) + const promise = Agni.search(addPaginationParams(esUrl, searchState), processQuery(jsonQuery, { searchState, getState })) .then(response => { if (skipProcessingFetch(getState, fetchingSearchIdx)) { promise.abort(); From b59f2db90fe537822e2840e6ec4aa88893717bc5 Mon Sep 17 00:00:00 2001 From: "Eugene Sypachev (@Axblade)" Date: Wed, 5 Jul 2017 19:31:19 +0300 Subject: [PATCH 61/61] get rid of posts to ES --- ashes/src/elastic/customer-groups.js | 6 +++--- ashes/src/elastic/customers.js | 8 ++++---- ashes/src/elastic/products.js | 6 +++--- ashes/src/elastic/promotions.js | 6 +++--- ashes/src/elastic/store-admins.js | 6 +++--- ashes/src/elastic/taxonomy.js | 7 +++---- ashes/src/modules/carts/payment-methods.js | 4 ++-- ashes/src/modules/gift-cards/new.js | 6 ++---- ashes/src/modules/live-search/searches-data.js | 3 ++- ashes/src/modules/quick-search.js | 4 ++-- ashes/src/modules/skus/suggest.js | 4 ++-- 11 files changed, 29 insertions(+), 31 deletions(-) diff --git a/ashes/src/elastic/customer-groups.js b/ashes/src/elastic/customer-groups.js index 3366d55a08..6affc1000a 100644 --- a/ashes/src/elastic/customer-groups.js +++ b/ashes/src/elastic/customer-groups.js @@ -1,10 +1,10 @@ /* @flow */ -import { post } from '../lib/search'; +import Agni from 'lib/agni'; import * as dsl from './dsl'; const MAX_RESULTS = 1000; -const searchUrl = `customer_groups_search_view/_search?size=${MAX_RESULTS}`; +const searchUrl = `customer_groups_search_view?size=${MAX_RESULTS}`; export function searchGroups(excludeGroups: Array, token: string) { let filters = []; @@ -31,5 +31,5 @@ export function searchGroups(excludeGroups: Array, token: string) { }, }); - return post(searchUrl, matchRule); + return Agni.search(searchUrl, matchRule); } diff --git a/ashes/src/elastic/customers.js b/ashes/src/elastic/customers.js index b90a496ecc..d716867b60 100644 --- a/ashes/src/elastic/customers.js +++ b/ashes/src/elastic/customers.js @@ -1,11 +1,11 @@ /* @flow */ import { toQuery } from './common'; -import { post } from '../lib/search'; +import Agni from 'lib/agni'; import * as dsl from './dsl'; const MAX_RESULTS = 1000; -const mapping = 'customers_search_view/_search'; +const mapping = 'customers_search_view'; const searchUrl = `${mapping}?size=${MAX_RESULTS}`; export function groupCount(criteria: Object, match: string) { @@ -17,7 +17,7 @@ export function groupSearch(criteria: Object, match: string, forCount: boolean = if (forCount) { req.size = 0; } - return post(mapping, req); + return Agni.search(mapping, req); } export function searchCustomers(excludes: Array, token: string) { @@ -45,5 +45,5 @@ export function searchCustomers(excludes: Array, token: string) { }, }); - return post(searchUrl, matchRule); + return Agni.search(searchUrl, matchRule); } diff --git a/ashes/src/elastic/products.js b/ashes/src/elastic/products.js index bc9ac1c439..a7e381357a 100644 --- a/ashes/src/elastic/products.js +++ b/ashes/src/elastic/products.js @@ -1,13 +1,13 @@ /* @flow */ -import { post } from '../lib/search'; +import Agni from 'lib/agni'; import * as dsl from './dsl'; import moment from 'moment'; // 1000 should be big enough to request all promotions with applyType = coupon // without size parameter ES responds with 10 items max const MAX_RESULTS = 1000; -const productsSearchUrl = `products_search_view/_search?size=${MAX_RESULTS}`; +const productsSearchUrl = `products_search_view?size=${MAX_RESULTS}`; type QueryOpts = { omitArchived: ?boolean, @@ -72,5 +72,5 @@ export function searchProducts(token: string, queryOpts: ?QueryOpts): Promise<*> }, }); - return post(productsSearchUrl, matchRule); + return Agni.search(productsSearchUrl, matchRule); } diff --git a/ashes/src/elastic/promotions.js b/ashes/src/elastic/promotions.js index 4ce8e86349..7f44ce7809 100644 --- a/ashes/src/elastic/promotions.js +++ b/ashes/src/elastic/promotions.js @@ -1,12 +1,12 @@ /* @flow */ -import { post } from '../lib/search'; +import Agni from 'lib/agni'; import * as dsl from './dsl'; // 1000 should be big enough to request all promotions with applyType = coupon // without size parameter ES responds with 10 items max const MAX_RESULTS = 1000; -const promotionsSearchUrl: string = `promotions_search_view/_search?size=${MAX_RESULTS}`; +const promotionsSearchUrl: string = `promotions_search_view?size=${MAX_RESULTS}`; export function searchCouponPromotions(token: string): Promise<*> { const filters = []; @@ -33,5 +33,5 @@ export function searchCouponPromotions(token: string): Promise<*> { }, }); - return post(promotionsSearchUrl, matchRule); + return Agni.search(promotionsSearchUrl, matchRule); } diff --git a/ashes/src/elastic/store-admins.js b/ashes/src/elastic/store-admins.js index 896b4a9ab0..cb441258dc 100644 --- a/ashes/src/elastic/store-admins.js +++ b/ashes/src/elastic/store-admins.js @@ -1,7 +1,7 @@ -import { post } from '../lib/search'; +import Agni from 'lib/agni'; import * as dsl from './dsl'; -const adminSearchUrl = 'store_admins_search_view/_search'; +const adminSearchUrl = 'store_admins_search_view'; export function searchAdmins(token) { const caseInsesnitiveToken = token.toLowerCase(); @@ -14,5 +14,5 @@ export function searchAdmins(token) { } }); - return post(adminSearchUrl, matchRule); + return Agni.search(adminSearchUrl, matchRule); } diff --git a/ashes/src/elastic/taxonomy.js b/ashes/src/elastic/taxonomy.js index b35cc6ea67..e256d86416 100644 --- a/ashes/src/elastic/taxonomy.js +++ b/ashes/src/elastic/taxonomy.js @@ -1,10 +1,10 @@ /* @flow */ -import { post } from '../lib/search'; +import Agni from 'lib/agni'; import * as dsl from './dsl'; const MAX_RESULTS = 1000; -const taxonomiesSearchUrl = `taxonomies_search_view/_search?size=${MAX_RESULTS}`; +const taxonomiesSearchUrl = `taxonomies_search_view?size=${MAX_RESULTS}`; export function searchTaxonomies(token: string): Promise<*> { const filters = []; @@ -26,6 +26,5 @@ export function searchTaxonomies(token: string): Promise<*> { }, }); - return post(taxonomiesSearchUrl, matchRule); + return Agni.search(taxonomiesSearchUrl, matchRule); } - diff --git a/ashes/src/modules/carts/payment-methods.js b/ashes/src/modules/carts/payment-methods.js index 00a4eebed7..34268a1c18 100644 --- a/ashes/src/modules/carts/payment-methods.js +++ b/ashes/src/modules/carts/payment-methods.js @@ -2,7 +2,7 @@ import _ from 'lodash'; import Api from 'lib/api'; import stripe from 'lib/stripe'; import { createAction, createReducer } from 'redux-act'; -import { post } from 'lib/search'; +import Agni from 'lib/agni'; import { getBillingAddress } from 'lib/credit-card-utils'; import { toQuery } from '../../elastic/common'; import { createAsyncActions } from '@foxcomm/wings'; @@ -75,7 +75,7 @@ const _giftCardSearch = createAsyncActions( }, }]; - return post('gift_cards_search_view/_search', toQuery(filters)); + return Agni.search('gift_cards_search_view', toQuery(filters)); } ); diff --git a/ashes/src/modules/gift-cards/new.js b/ashes/src/modules/gift-cards/new.js index 92e3ede4d5..8c93fc89d7 100644 --- a/ashes/src/modules/gift-cards/new.js +++ b/ashes/src/modules/gift-cards/new.js @@ -1,15 +1,13 @@ - /* @flow weak */ // state for gift card adding form import _ from 'lodash'; -import Api from '../../lib/api'; +import Api from 'lib/api'; import { combineReducers } from 'redux'; import { createAction, createReducer } from 'redux-act'; import { assoc } from 'sprout-data'; - const _createAction = (desc, ...args) => createAction(`GIFT_CARDS_NEW_${desc}`, ...args); export const changeFormData = _createAction('CHANGE_FORM', (name, value) => ({name, value})); @@ -26,7 +24,7 @@ const emptyFilters = []; const emptyPhrase = ''; const quickSearch = makeQuickSearch( 'giftCards.adding.suggestedCustomers', - 'customers_search_view/_search', + 'customers_search_view', emptyFilters, emptyPhrase ); diff --git a/ashes/src/modules/live-search/searches-data.js b/ashes/src/modules/live-search/searches-data.js index 5223817c68..861a039405 100644 --- a/ashes/src/modules/live-search/searches-data.js +++ b/ashes/src/modules/live-search/searches-data.js @@ -59,7 +59,8 @@ export default function makeDataInSearches(namespace, esUrl, options = {}) { dispatch(saveRawQuery(jsonQuery)); - const promise = Agni.search(addPaginationParams(esUrl, searchState), processQuery(jsonQuery, { searchState, getState })) + const promise = + Agni.search(addPaginationParams(esUrl, searchState), processQuery(jsonQuery, { searchState, getState })) .then(response => { if (skipProcessingFetch(getState, fetchingSearchIdx)) { promise.abort(); diff --git a/ashes/src/modules/quick-search.js b/ashes/src/modules/quick-search.js index 7d2acd7ea0..4d03b2a714 100644 --- a/ashes/src/modules/quick-search.js +++ b/ashes/src/modules/quick-search.js @@ -1,6 +1,6 @@ import _ from 'lodash'; import { createReducer } from 'redux-act'; -import { post } from '../lib/search'; +import Agni from 'lib/search'; import { update } from 'sprout-data'; import { toQuery } from '../elastic/common'; import SearchTerm from '../paragons/search-term'; @@ -38,7 +38,7 @@ export default function makeQuickSearch(namespace, searchUrl, searchFilters, phr function fetcher(phrase, queryFilters = filters, options = {}) { options.phrase = phrase; const esQuery = toQuery(queryFilters, options); - return post(addPaginationParams(url, this.searchState), esQuery); + return Agni.search(addPaginationParams(url, this.searchState), esQuery); } const {reducer, ...actions} = makePagination(namespace, fetcher, state => _.get(state, `${namespace}.results`)); diff --git a/ashes/src/modules/skus/suggest.js b/ashes/src/modules/skus/suggest.js index 0e85bb58e3..ae12b0e9fe 100644 --- a/ashes/src/modules/skus/suggest.js +++ b/ashes/src/modules/skus/suggest.js @@ -3,7 +3,7 @@ import _ from 'lodash'; import { createAction, createReducer } from 'redux-act'; -import { post } from 'lib/search'; +import Agni from 'lib/agni'; import * as dsl from 'elastic/dsl'; import { createAsyncActions } from '@foxcomm/wings'; @@ -30,7 +30,7 @@ const _suggestSkus = createAsyncActions( })]; } - return post('sku_search_view/_search', dsl.query({ + return Agni.search('sku_search_view', dsl.query({ bool: { filter: filters, should: [