Skip to content

Commit

Permalink
Rename packages to ...iskra
Browse files Browse the repository at this point in the history
  • Loading branch information
prolativ committed Aug 30, 2022
1 parent acd4924 commit e3d972f
Show file tree
Hide file tree
Showing 30 changed files with 67 additions and 67 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Iskra tries to work around this problem by using its own encoders (unrelated to

## How does Iskra make things typesafe and efficient at the same time?

TypedFrames provides thin (but strongly typed) wrappers around `DataFrame`s, which track types and names of columns at compile time but let Catalyst perform all of its optimizations at runtime.
Iskra provides thin (but strongly typed) wrappers around `DataFrame`s, which track types and names of columns at compile time but let Catalyst perform all of its optimizations at runtime.

Iskra uses structural types rather than case classes as data models, which gives us a lot of flexibility (no need to explicitly define a new case class when a column is added/removed/renamed!) but we still get compilation errors when we try to refer to a column which doesn't exist or can't be used in a given context.

Expand Down
6 changes: 3 additions & 3 deletions publish-conf.scala
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
//> using publish.organization "org.virtuslab"
//> using publish.name "typed-frames"
//> using publish.name "iskra"
//> using publish.computeVersion "git:tag"
//> using publish.url "https://github.com/VirtusLab/typed-frames"
//> using publish.versionControl "scm:git:github.com/VirtusLab/typed-frames.git"
//> using publish.url "https://github.com/VirtusLab/iskra"
//> using publish.versionControl "scm:git:github.com/VirtusLab/iskra.git"
//> using publish.license "Apache-2.0"
//> using publish.repository "central"
//> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ"
2 changes: 1 addition & 1 deletion src/main/Aliasing.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*

Expand Down
2 changes: 1 addition & 1 deletion src/main/Column.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import org.apache.spark.sql.{ Column => UntypedColumn}
import types.DataType
Expand Down
8 changes: 4 additions & 4 deletions src/main/ColumnOp.scala
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*
import org.apache.spark.sql
import org.apache.spark.sql.functions.concat
import org.virtuslab.typedframes.{Column as Col}
import org.virtuslab.typedframes.UntypedOps.typed
import org.virtuslab.typedframes.types.*
import org.virtuslab.iskra.{Column as Col}
import org.virtuslab.iskra.UntypedOps.typed
import org.virtuslab.iskra.types.*
import DataType.*

object ColumnOp:
Expand Down
2 changes: 1 addition & 1 deletion src/main/DataFrame.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import org.apache.spark.sql
import org.apache.spark.sql.SparkSession
Expand Down
6 changes: 3 additions & 3 deletions src/main/DataFrameBuilders.scala
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted._
import org.apache.spark.sql
import org.apache.spark.sql.SparkSession
import org.virtuslab.typedframes.DataFrame
import org.virtuslab.typedframes.types.{DataType, StructType}
import org.virtuslab.iskra.DataFrame
import org.virtuslab.iskra.types.{DataType, StructType}
import DataType.{Encoder, StructEncoder, PrimitiveEncoder}

object DataFrameBuilders:
Expand Down
2 changes: 1 addition & 1 deletion src/main/FrameSchema.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*
import scala.deriving.Mirror
Expand Down
4 changes: 2 additions & 2 deletions src/main/Grouping.scala
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*
import org.virtuslab.typedframes.types.DataType
import org.virtuslab.iskra.types.DataType
import MacroHelpers.TupleSubtype

class GroupBy[View <: SchemaView](val view: View, val underlying: UntypedDataFrame)
Expand Down
2 changes: 1 addition & 1 deletion src/main/Join.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*

Expand Down
4 changes: 2 additions & 2 deletions src/main/JoinOnCondition.scala
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*
import org.virtuslab.typedframes.types.BooleanOptType
import org.virtuslab.iskra.types.BooleanOptType

abstract class JoinOnCondition[MergedSchema, MergedView <: SchemaView](
val left: UntypedDataFrame, val right: UntypedDataFrame, val joinType: JoinType
Expand Down
4 changes: 2 additions & 2 deletions src/main/MacroHelpers.scala
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*

private[typedframes] object MacroHelpers:
private[iskra] object MacroHelpers:
def callPosition(ownerExpr: Expr[?])(using Quotes): quotes.reflect.Position =
import quotes.reflect.*
val file = Position.ofMacroExpansion.sourceFile
Expand Down
2 changes: 1 addition & 1 deletion src/main/Name.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

type Name = String & Singleton
object Name:
Expand Down
2 changes: 1 addition & 1 deletion src/main/SchemaView.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted._
import org.apache.spark.sql.functions.col
Expand Down
2 changes: 1 addition & 1 deletion src/main/Select.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*
import MacroHelpers.TupleSubtype
Expand Down
2 changes: 1 addition & 1 deletion src/main/UntypedOps.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*
import types.{DataType, StructType}
Expand Down
4 changes: 2 additions & 2 deletions src/main/Where.scala
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

import scala.quoted.*
import scala.annotation.implicitNotFound

import org.virtuslab.typedframes.types.BooleanOptType
import org.virtuslab.iskra.types.BooleanOptType

trait Where[Schema, View <: SchemaView]:
val view: View
Expand Down
10 changes: 5 additions & 5 deletions src/main/api/api.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra
package api

export DataFrameBuilders.toTypedDF
Expand All @@ -24,11 +24,11 @@ export types.{
StructOptType
}
export UntypedOps.typed
export org.virtuslab.typedframes.$
export org.virtuslab.typedframes.{Column, DataFrame, UntypedColumn, UntypedDataFrame}
export org.virtuslab.iskra.$
export org.virtuslab.iskra.{Column, DataFrame, UntypedColumn, UntypedDataFrame}

object functions:
export org.virtuslab.typedframes.functions.lit
export org.virtuslab.typedframes.functions.Aggregates.*
export org.virtuslab.iskra.functions.lit
export org.virtuslab.iskra.functions.Aggregates.*

export org.apache.spark.sql.SparkSession
12 changes: 6 additions & 6 deletions src/main/functions/aggregates.scala
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package org.virtuslab.typedframes.functions
package org.virtuslab.iskra.functions

import org.apache.spark.sql
import org.virtuslab.typedframes.Agg
import org.virtuslab.typedframes.Column
import org.virtuslab.typedframes.UntypedOps.typed
import org.virtuslab.typedframes.types.*
import org.virtuslab.typedframes.types.DataType.{NumericOptType, Nullable}
import org.virtuslab.iskra.Agg
import org.virtuslab.iskra.Column
import org.virtuslab.iskra.UntypedOps.typed
import org.virtuslab.iskra.types.*
import org.virtuslab.iskra.types.DataType.{NumericOptType, Nullable}

class Sum[A <: Agg](val agg: A):
def apply[T <: NumericOptType](column: agg.View ?=> Column[T]): Column[Nullable[T]] =
Expand Down
6 changes: 3 additions & 3 deletions src/main/functions/lit.scala
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.virtuslab.typedframes.functions
package org.virtuslab.iskra.functions

import org.apache.spark.sql
import org.virtuslab.typedframes.Column
import org.virtuslab.typedframes.types.DataType.PrimitiveEncoder
import org.virtuslab.iskra.Column
import org.virtuslab.iskra.types.DataType.PrimitiveEncoder

def lit[A](value: A)(using encoder: PrimitiveEncoder[A]): Column[encoder.ColumnType] = Column(sql.functions.lit(encoder.encode(value)))
2 changes: 1 addition & 1 deletion src/main/types/DataType.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra
package types

import scala.quoted._
Expand Down
2 changes: 1 addition & 1 deletion src/main/untyped.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.virtuslab.typedframes
package org.virtuslab.iskra

type UntypedDataFrame = org.apache.spark.sql.DataFrame
type UntypedColumn = org.apache.spark.sql.Column
Expand Down
4 changes: 2 additions & 2 deletions src/test/AggregatorsTest.scala
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.virtuslab.typedframes.test
package org.virtuslab.iskra.test

class AggregatorsTest extends SparkUnitTest("AggregatorsTest"):
import org.virtuslab.typedframes.api.*
import org.virtuslab.iskra.api.*
import functions.*

case class Foo(string: String, int: Int, intOpt: Option[Int], float: Float, floatOpt: Option[Float])
Expand Down
18 changes: 9 additions & 9 deletions src/test/CompilationTest.scala
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
package org.virtuslab.typedframes.test
package org.virtuslab.iskra.test

import org.scalatest.funsuite.AnyFunSuite

class CompilationTest extends AnyFunSuite:
test("Wrong collect type") {
assertCompiles("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|given spark: SparkSession = ???
|val elements = Seq(1, 2, 3).toTypedDF.collectAs[Int]
|""".stripMargin)

assertDoesNotCompile("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|given spark: SparkSession = ???
|val elements = Seq(1, 2, 3).toTypedDF.collectAs[String]
|""".stripMargin)
}

test("Select nonexistent column") {
assertCompiles("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|case class Foo(string: String)
|given spark: SparkSession = ???
|val elements = Seq(Foo("abc")).toTypedDF.select($.string)
|""".stripMargin)

assertDoesNotCompile("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|case class Foo(string: String)
|given spark: SparkSession = ???
|val elements = Seq(Foo("abc")).toTypedDF.select($.strin)
Expand All @@ -35,25 +35,25 @@ class CompilationTest extends AnyFunSuite:

test("Collect nullable") {
assertCompiles("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|given spark: SparkSession = ???
|val result = Seq(1, 2).toTypedDF.collectAs[Int]
|""".stripMargin)

assertCompiles("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|given spark: SparkSession = ???
|val result = Seq(1, 2).toTypedDF.collectAs[Option[Int]]
|""".stripMargin)

assertDoesNotCompile("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|given spark: SparkSession = ???
|val result = Seq(Some(1), None).toTypedDF.collectAs[Int]
|""".stripMargin)

assertCompiles("""
|import org.virtuslab.typedframes.api.*
|import org.virtuslab.iskra.api.*
|given spark: SparkSession = ???
|val result = Seq(Some(1), None).toTypedDF.collectAs[Option[Int]]
|""".stripMargin)
Expand Down
4 changes: 2 additions & 2 deletions src/test/OperatorsTest.scala
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package org.virtuslab.typedframes.test
package org.virtuslab.iskra.test

import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.BeforeAndAfterAll
import org.scalatest.matchers.should.Matchers.shouldEqual

class OperatorsTest extends SparkUnitTest("OperatorsTest"):
import org.virtuslab.typedframes.api.*
import org.virtuslab.iskra.api.*

case class Foo(boolean: Boolean, string: String, byte: Byte, short: Short, int: Int, long: Long, float: Float, double: Double)
case class Bar(int: Int, intSome: Option[Int], intNone: Option[Int])
Expand Down
4 changes: 2 additions & 2 deletions src/test/SparkUnitTest.scala
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package org.virtuslab.typedframes.test
package org.virtuslab.iskra.test

import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.BeforeAndAfterAll
import org.virtuslab.typedframes.api.*
import org.virtuslab.iskra.api.*

abstract class SparkUnitTest(suiteName: String) extends AnyFunSuite, BeforeAndAfterAll:
given spark: SparkSession =
Expand Down
4 changes: 2 additions & 2 deletions src/test/WhereTest.scala
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package org.virtuslab.typedframes.test
package org.virtuslab.iskra.test

import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.BeforeAndAfterAll
import org.scalatest.matchers.should.Matchers.shouldEqual

class WhereTest extends SparkUnitTest("WhereTest"):
import org.virtuslab.typedframes.api.*
import org.virtuslab.iskra.api.*
import functions.lit

case class Foo(int: Int, intOpt: Option[Int])
Expand Down
4 changes: 2 additions & 2 deletions src/test/example/Books.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.virtuslab.typedframes.example.books
package org.virtuslab.iskra.example.books

import org.virtuslab.typedframes.api.*
import org.virtuslab.iskra.api.*

@main def runExample(dataFilePath: String): Unit =
given spark: SparkSession = {
Expand Down
4 changes: 2 additions & 2 deletions src/test/example/Countries.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.virtuslab.typedframes.example.countries
package org.virtuslab.iskra.example.countries

import org.virtuslab.typedframes.api.*
import org.virtuslab.iskra.api.*
import functions.avg

@main def runExample(): Unit =
Expand Down
4 changes: 2 additions & 2 deletions src/test/example/Workers.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.virtuslab.typedframes.example.workers
package org.virtuslab.iskra.example.workers

import org.virtuslab.typedframes.api.*
import org.virtuslab.iskra.api.*
import functions.lit

@main def runExample(): Unit =
Expand Down

0 comments on commit e3d972f

Please sign in to comment.