-
Notifications
You must be signed in to change notification settings - Fork 0
/
rel-nifi.yaml
323 lines (316 loc) · 14 KB
/
rel-nifi.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
apiVersion: nifi.orange.com/v1alpha1
kind: NifiCluster
metadata:
name: mynifi
spec:
service:
# headlessEnabled specifies if the cluster should use headlessService for Nifi or individual services
# using service per nodes may come an handy case of service mesh.
headlessEnabled: true
# Annotations specifies the annotations to attach to services the operator creates
# annotations:
# pod:
# Annotations specifies the annotations to attach to pods the operator creates
# annotations:
# zKAddress specifies the ZooKeeper connection string
# in the form hostname:port where host and port are those of a Zookeeper server.
zkAddress: "zookeeper.zookeeper:2181"
# zKPath specifies the Zookeeper chroot path as part
# of its Zookeeper connection string which puts its data under same path in the global ZooKeeper namespace.
zkPath: "/myNiFi"
# initContainerImage can override the default image used into the init container to check if
# ZoooKeeper server is reachable.
# initContainerImage: busybox
# initContainers defines additional initContainers configurations
initContainers:
- image: eu.gcr.io/poc-rtc/nifi-proxydata-bundle:1.11.4-1.0.0
name: nifi-proxydata-lib
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "cp -vr /nifi_lib/* /nifi_lib_volume/"]
volumeMounts:
- name: extensions-repository
mountPath: /nifi_lib_volume
- name: logs
mountPath: /logs
# clusterImage can specify the whole nificluster image in one place
clusterImage: "apache/nifi:1.12.1"
# oneNifiNodePerNode if set to true every nifi node is started on a new node, if there is not enough node to do that
# it will stay in pending state. If set to false the operator also tries to schedule the nifi node to a unique node
# but if the node number is insufficient the nifi node will be scheduled to a node where a nifi node is already running.
oneNifiNodePerNode: false
#
propagateLabels: true
# managedAdminUsers:
# - identity : "[email protected]"
# name: "aguitton"
# managedReaderUsers:
# - identity : "[email protected]"
# name: "toto"
# readOnlyConfig specifies the read-only type Nifi config cluster wide, all theses
# will be merged with node specified readOnly configurations, so it can be overwritten per node.
readOnlyConfig:
# MaximumTimerDrivenThreadCount define the maximum number of threads for timer driven processors available to the system.
maximumTimerDrivenThreadCount: 30
# Logback configuration that will be applied to the node
logbackConfig:
# logback.xml configuration that will replace the one produced based on template
replaceConfigMap:
# The key of the value,in data content, that we want use.
data: logback.xml
# Name of the configmap that we want to refer.
name: nifi-config
# Namespace where is located the secret that we want to refer.
namespace: nifi
# NifiProperties configuration that will be applied to the node.
nifiProperties:
# Additionnals nifi.properties configuration that will override the one produced based on template and
# configuration
overrideConfigMap:
# The key of the value,in data content, that we want use.
data: nifi.properties
# Name of the configmap that we want to refer.
name: nifi-config
# Namespace where is located the secret that we want to refer.
namespace: nifi
# A comma separated list of allowed HTTP Host header values to consider when NiFi
# is running securely and will be receiving requests to a different host[:port] than it is bound to.
# https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties
# webProxyHosts:
# Nifi security client auth
needClientAuth: false
# Indicates which of the configured authorizers in the authorizers.xml file to use
# https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration
# authorizer:
# ZookeeperProperties configuration that will be applied to the node.
zookeeperProperties:
# Additionnals zookeeeper.properties configuration that will override the one produced based on template and
# configuration
overrideConfigMap:
# The key of the value,in data content, that we want use.
data: zookeeper.properties
# Name of the configmap that we want to refer.
name: nifi-config
# Namespace where is located the secret that we want to refer.
namespace: nifi
bootstrapProperties:
# Additionnals bootstrap.properties configuration that will override the one produced based on template and
# configuration
overrideConfigMap:
# The key of the value,in data content, that we want use.
data: bootstrap.conf
# Name of the configmap that we want to refer.
name: nifi-config
# Namespace where is located the secret that we want to refer.
namespace: nifi
# JVM memory settings
nifiJvmMemory: "512m"
# Additionnals bootstrap.properties configuration that will override the one produced based
# on template and configurations.
# https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#bootstrap_properties
overrideConfigs: |
java.arg.4=-Djava.net.preferIPv4Stack=true
# nodeConfigGroups specifies multiple node configs with unique name
nodeConfigGroups:
default_group:
# provenanceStorage allow to specify the maximum amount of data provenance information to store at a time
# https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties
provenanceStorage: "10 GB"
#RunAsUser define the id of the user to run in the Nifi image
# +kubebuilder:validation:Minimum=1
runAsUser: 1000
# Set this to true if the instance is a node in a cluster.
# https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup
isNode: true
# Docker image used by the operator to create the node associated
# https://hub.docker.com/r/apache/nifi/
# image: "apache/nifi:1.11.2"
# nodeAffinity can be specified, operator populates this value if new pvc added later to node
# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity
# nodeAffinity:
# imagePullPolicy define the pull policy for NiFi cluster docker image
imagePullPolicy: IfNotPresent
# storageConfigs specifies the node related configs
storageConfigs:
# Name of the storage config, used to name PV to reuse into sidecars for example.
- name: provenance-repository
# Path where the volume will be mount into the main nifi container inside the pod.
mountPath: "/opt/nifi/provenance_repository"
# Kubernetes PVC spec
# https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim
pvcSpec:
accessModes:
- ReadWriteMany
storageClassName: "ibmc-file-silver"
resources:
requests:
storage: 10Gi
- mountPath: "/opt/nifi/nifi-current/logs"
name: logs
pvcSpec:
accessModes:
- ReadWriteMany
storageClassName: "ibmc-file-silver"
resources:
requests:
storage: 10Gi
# serviceAccountName specifies the serviceAccount used for this specific node
serviceAccountName: "default"
# resourceRequirements works exactly like Container resources, the user can specify the limit and the requests
# through this property
# https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
resourcesRequirements:
limits:
cpu: "2"
memory: 3Gi
requests:
cpu: "1"
memory: 1Gi
# imagePullSecrets specifies the secret to use when using private registry
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#localobjectreference-v1-core
# imagePullSecrets: devops-registry-secret
# nodeSelector can be specified, which set the pod to fit on a node
# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# nodeSelector:
# tolerations can be specified, which set the pod's tolerations
# https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#concepts
# tolerations:
# Additionnal annotation to attach to the pod associated
# https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
# nodeAnnotations:
# disruptionBudget defines the configuration for PodDisruptionBudget
disruptionBudget:
# create will enable the PodDisruptionBudget when set to true
create: false
# The budget to set for the PDB, can either be static number or a percentage
# budget: "1"
# all node requires an image, unique id, and storageConfigs settings
nodes:
# Unique Node id
- id: 0
# nodeConfigGroup can be used to ease the node configuration, if set only the id is required
nodeConfigGroup: "default_group"
# readOnlyConfig can be used to pass Nifi node config
# which has type read-only these config changes will trigger rolling upgrade
readOnlyConfig:
nifiProperties:
overrideConfigs: |
nifi.ui.banner.text=NiFiKop by Orange - Node 0
# node configuration
# nodeConfig:
- id: 1
# nodeConfigGroup can be used to ease the node configuration, if set only the id is required
nodeConfigGroup: "default_group"
# readOnlyConfig can be used to pass Nifi node config
# which has type read-only these config changes will trigger rolling upgrade
readOnlyConfig:
nifiProperties:
overrideConfigs: |
nifi.ui.banner.text=NiFiKop by Orange - Node 1
# node configuration
# nodeConfig:
- id: 2
# readOnlyConfig can be used to pass Nifi node config
# which has type read-only these config changes will trigger rolling upgrade
nodeConfigGroup: "default_group"
readOnlyConfig:
overrideConfigs: |
nifi.ui.banner.text=NiFiKop by Orange - Node 2
# LdapConfiguration specifies the configuration if you want to use LDAP
ldapConfiguration:
# If set to true, we will enable ldap usage into nifi.properties configuration.
enabled: true
# Space-separated list of URLs of the LDAP servers (i.e. ldap://<hostname>:<port>).
url: ldaps://bluepages.ibm.com:636
# Base DN for searching for users (i.e. CN=Users,DC=example,DC=com).
searchBase: ou=bluepages,o=ibm.com
# Filter for searching for users against the 'User Search Base'.
# (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'.
searchFilter: (mail={0})
groupSearchBase: ou=memberlist,ou=ibmgroups,o=ibm.com
groupObjectClass: groupOfUniqueNames
groupSearchFilter: (cn=Optimizer_SVT*)
groupMemberAttribute: uniquemember
groupNameAttribute: cn
admin: uid=AVTMH6672,c=cn,ou=bluepages,o=ibm.com
initUser: cn=Optimizer_SVT_Write,ou=memberlist,ou=ibmgroups,o=ibm.com
# NifiClusterTaskSpec specifies the configuration of the nifi cluster Tasks
nifiClusterTaskSpec:
# retryDurationMinutes describes the amount of time the Operator waits for the task
retryDurationMinutes: 10
# listenerConfig specifies nifi's listener specifig configs
listenersConfig:
# internalListeners specifies settings required to access nifi internally
internalListeners:
# (Optional field) Type allow to specify if we are in a specific nifi listener
# it's allowing to define some required information such as Cluster Port,
# Http Port, Https Port or S2S port
# {"cluster", "http", "https", "s2s"}
- type: "https"
# An identifier for the port which will be configured.
name: "https"
# The container port.
containerPort: 8080
- type: "cluster"
name: "cluster"
containerPort: 6007
- type: "s2s"
name: "s2s"
containerPort: 10000
- name: "prometheus"
containerPort: 9020
- name: "rsyslog"
containerPort: 10001
- type: "prometheus"
name: "prometheus"
containerPort: 9090
# sslSecrets contains information about ssl related kubernetes secrets if one of the
# listener setting type set to ssl these fields must be populated to
sslSecrets:
tlsSecretName: "nifi-ssl-tls-secret"
# tlsSecretName: "test-nifikop"
# create tells the installed cert manager to create the required certs keys
create: true
clusterScoped: false
# clusterScoped: false
# issuerRef allow to use an existing issuer to act as CA :
# https://cert-manager.io/docs/concepts/issuer/ to-do
# issuerRef:
# name: letsencrypt-staging
# kind: Issuer
externalServices:
- name: "clusterip"
spec:
type: ClusterIP
portConfigs:
- port: 8080
internalListenerName: "https"
serviceAnnotations:
toto: tata
sidecarConfigs:
- name: app-log
image: "busybox:1.32.0"
args: [ tail, -n+1, -F, /var/log/nifi-app.log ]
resources: &log_resources
requests:
cpu: 50m
memory: 50Mi
limits:
cpu: 50m
memory: 50Mi
volumeMounts:
- name: logs
mountPath: /var/log
- name: bootstrap-log
image: "busybox:1.32.0"
args: [tail, -n+1, -F, /var/log/nifi-bootstrap.log]
resources: *log_resources
volumeMounts:
- name: logs
mountPath: /var/log
- name: user-log
image: "busybox:1.32.0"
args: [tail, -n+1, -F, /var/log/nifi-user.log]
resources: *log_resources
volumeMounts:
- name: logs
mountPath: /var/log