-
Notifications
You must be signed in to change notification settings - Fork 6
/
frameworks-versions.properties
194 lines (170 loc) · 7.15 KB
/
frameworks-versions.properties
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
#
# Copyright (C) 2022 Dremio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Spark + Scala
# Known Spark major version
sparkVersions=3.1,3.2,3.3,3.4,3.5
sparkDefaultVersion=3.5
# Supported Scala major versions, by Spark major version
sparkVersion-3.1-scalaVersions=2.12
sparkVersion-3.2-scalaVersions=2.12,2.13
sparkVersion-3.3-scalaVersions=2.12,2.13
sparkVersion-3.4-scalaVersions=2.12,2.13
sparkVersion-3.5-scalaVersions=2.12,2.13
# Exact Spark versions by major version
versionSpark-3.1=3.1.3
versionSpark-3.2=3.2.4
versionSpark-3.3=3.3.4
versionSpark-3.4=3.4.3
versionSpark-3.5=3.5.3
# Known Scala major versions
scalaVersions=2.12,2.13
scalaDefaultVersion=2.12
# Exact Scala versions by major version
versionScala-2.12=2.12.20
versionScala-2.13=2.13.15
#
# Flink versions
# Known Flink major versions
flinkVersions=1.14,1.15,1.16,1.17,1.18,1.19,1.20
flinkDefaultVersion=1.18
# Exact flink version by major version
versionFlink-1.14=1.14.6
versionFlink-1.15=1.15.4
versionFlink-1.16=1.16.3
versionFlink-1.17=1.17.2
versionFlink-1.18=1.18.1
versionFlink-1.19=1.19.0
versionFlink-1.20=1.20.0
# hadoop version for Flink (some Hadoop dependencies required by Flink)
versionFlinkHadoop-1.14=2.7.3
versionFlinkHadoop-1.15=2.7.3
versionFlinkHadoop-1.16=2.7.3
versionFlinkHadoop-1.17=2.7.3
versionFlinkHadoop-1.18=2.7.3
versionFlinkHadoop-1.19=2.7.3
versionFlinkHadoop-1.20=2.7.3
# Scala versions for Flink, not really using Scala for Flink, but some Java dependencies have Scala
# version numbers in their artifact-ID.
flink-scala-1.14=2.12
flink-scala-1.15=2.12
flink-scala-1.16=2.12
flink-scala-1.17=2.12
flink-scala-1.18=2.12
flink-scala-1.19=2.12
flink-scala-1.20=2.12
flink-scalaForDependencies-1.14=_2.12
flink-scalaForDependencies-1.15=
flink-scalaForDependencies-1.16=
flink-scalaForDependencies-1.17=
flink-scalaForDependencies-1.18=
flink-scalaForDependencies-1.19=
flink-scalaForDependencies-1.20=
#
# Presto versions
# Known Presto major versions
prestoVersions=0.274,0.275,0.276,0.277,0.278,0.279,0.280,0.281
prestoDefaultVersion=0.281
# Exact Presto versions by Presto major versions
versionPresto-0.274=0.274
versionPresto-0.275=0.275
versionPresto-0.276=0.276.2
versionPresto-0.277=0.277
versionPresto-0.278=0.278.1
versionPresto-0.279=0.279
versionPresto-0.280=0.280
versionPresto-0.281=0.281
#
# Define the Cross-Engine setups
crossEngineSetups=spark32flink114,spark33flink115,spark33flink116,spark33flink117,spark35flink118
# TODO ideally we'd like to run Spark 3.4 with Flink 1.17, but that does not work, because there's
# some awful dependency issue with `org.codehaus.janino.InternalCompilerException` coming from
# org.codehaus.janino:janino (Flink 1.17.x) vs org.codehaus.janino:commons-compiler (Spark 3.4),
# that manifests as:
# java.lang.SecurityException: class "org.codehaus.commons.compiler.InternalCompilerException"'s signer information does not match signer information of other classes in the same package
# So we cannot use the 'spark34flink117'
# TODO ideally we'd like to run Spark 3.5 with Flink 1.17, but that does not work, because there's
# a dependency conflict with Flink 1.17.1 + Spark 3.5, requiring different versions of
# org.apache.logging.log4j:log4j-slf4j2-impl leading to
# 'java.lang.NoSuchMethodError: 'void org.apache.logging.slf4j.Log4jLoggerFactory.<init>(org.apache.logging.slf4j.Log4jMarkerFactory)'
# via Flink.
crossEngine.spark32flink114.sparkMajorVersion=3.2
crossEngine.spark32flink114.scalaMajorVersion=2.12
crossEngine.spark32flink114.flinkMajorVersion=1.14
crossEngine.spark33flink115.sparkMajorVersion=3.3
crossEngine.spark33flink115.scalaMajorVersion=2.12
crossEngine.spark33flink115.flinkMajorVersion=1.15
crossEngine.spark33flink116.sparkMajorVersion=3.3
crossEngine.spark33flink116.scalaMajorVersion=2.12
crossEngine.spark33flink116.flinkMajorVersion=1.16
crossEngine.spark33flink117.sparkMajorVersion=3.3
crossEngine.spark33flink117.scalaMajorVersion=2.12
crossEngine.spark33flink117.flinkMajorVersion=1.17
crossEngine.spark34flink117.sparkMajorVersion=3.4
crossEngine.spark34flink117.scalaMajorVersion=2.12
crossEngine.spark34flink117.flinkMajorVersion=1.17
crossEngine.spark35flink117.sparkMajorVersion=3.5
crossEngine.spark35flink117.scalaMajorVersion=2.12
crossEngine.spark35flink117.flinkMajorVersion=1.17
crossEngine.spark35flink118.sparkMajorVersion=3.5
crossEngine.spark35flink118.scalaMajorVersion=2.12
crossEngine.spark35flink118.flinkMajorVersion=1.18
crossEngine.spark35flink119.sparkMajorVersion=3.5
crossEngine.spark35flink119.scalaMajorVersion=2.12
crossEngine.spark35flink119.flinkMajorVersion=1.19
crossEngine.spark35flink120.sparkMajorVersion=3.5
crossEngine.spark35flink120.scalaMajorVersion=2.12
crossEngine.spark35flink120.flinkMajorVersion=1.20
#
# Following properties define a bunch of major-version restrictions.
# This helps modelling the produced Gradle projects depending on the versions of e.g. Nessie or
# Iceberg being used.
# TODO constraints.sparkVersions.nessie-0.30=3.1,3.2 --> there are issues w/ Spark 3.1
# Spark version restrictions - for specific Nessie versions
constraints.sparkVersions.nessie-0.58=3.1,3.2,3.3
constraints.sparkVersions.nessie-0.59=3.1,3.2,3.3
constraints.sparkVersions.nessie-0.60=3.1,3.2,3.3
constraints.sparkVersions.nessie-0.71=3.1,3.2,3.3,3.4
constraints.sparkVersions.nessie-0.78=3.2,3.3,3.4,3.5
constraints.sparkVersions.nessie-0.79=3.3,3.4,3.5
# Spark version restrictions - for specific Iceberg versions
constraints.sparkVersions.iceberg-1.1=3.1,3.2,3.3
constraints.sparkVersions.iceberg-1.2=3.1,3.2,3.3
constraints.sparkVersions.iceberg-1.3=3.1,3.2,3.3,3.4
constraints.sparkVersions.iceberg-1.4=3.2,3.3,3.4,3.5
constraints.sparkVersions.iceberg-1.5=3.3,3.4,3.5
constraints.sparkVersions.iceberg-1.6=3.3,3.4,3.5
constraints.sparkVersions.iceberg-999.99=3.3,3.4,3.5
# Flink version restrictions - for specific Iceberg versions
constraints.flinkVersions.iceberg-1.0=1.14,1.15
constraints.flinkVersions.iceberg-1.1=1.14,1.15,1.16
constraints.flinkVersions.iceberg-1.2=1.14,1.15,1.16
constraints.flinkVersions.iceberg-1.3=1.15,1.16,1.17
constraints.flinkVersions.iceberg-1.4=1.15,1.16,1.17
constraints.flinkVersions.iceberg-1.5=1.16,1.17,1.18
constraints.flinkVersions.iceberg-1.6=1.17,1.18,1.19
constraints.flinkVersions.iceberg-999.99=1.18,1.19,1.20
# Presto version restrictions - for specific Iceberg versions
constraints.prestoVersions.iceberg-1.0=0.277
constraints.prestoVersions.iceberg-1.1=0.278
constraints.prestoVersions.iceberg-1.2=0.281
constraints.prestoVersions.iceberg-1.3=0.281
constraints.prestoVersions.iceberg-1.4=0.281
constraints.prestoVersions.iceberg-1.5=0.281
constraints.prestoVersions.iceberg-1.6=0.281
# Presto version supporting current "main branch" Iceberg
# '-' means no Presto
constraints.prestoVersions.iceberg-999.99=-