-
Notifications
You must be signed in to change notification settings - Fork 124
/
example.yml
149 lines (140 loc) · 5.46 KB
/
example.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
inputs:
# - PlainFile:
# path:
# - /tmp/file/^file.*\.txt$
# heartbeatHost: 172.16.1.52:8854
# userToken: kRhisbdoTQMCZU5KqQqGkQ7sDA7BM9kpldnQ5Nf2al8ER9yp
# offsetDbSyncIntervalMs: 60000
# heartbeatIntevalMs: 60000
- Kafka:
codec: json
encoding: UTF8 # defaut UTF8
topic:
{"dt_all_log": 1}
consumerSettings:
group.id: dt_all_log_group_hao
zookeeper.connect: 172.16.1.181:2181,172.16.1.186:2181,172.16.1.226:2181/kafka2
auto.commit.interval.ms: "1000"
# - KafkaReset:
# codec: json
# encoding: UTF8 # defaut UTF8
# minTime: "2017-08-11 20:33:33"
# maxTime: "2017-08-11 20:34:33"
# topic:
# {"dt_all_log": 3}
# consumerSettings:
# group.id: dt_all_log_group_hao
# zookeeper.connect: 172.16.1.181:2181,172.16.1.186:2181,172.16.1.226:2181/kafka2
# auto.commit.interval.ms: "1000"
# - Netty:
# port: 8635
# whiteListPath:
# codec: json
# isExtract: false
filters:
- Performance:
interval: 1
path: /tmp/filter-performance-%{+YYYY.MM.dd}.txt
timeZone: Asia/Shanghai
monitorPath: {"/tmp/filter-performance-%{+YYYY.MM.dd}.txt":"8"}
outputs:
- Performance:
interval: 1
path: /tmp/beat-performance-%{+YYYY.MM.dd}.txt
timeZone: Asia/Shanghai
monitorPath: {"/tmp/beat-performance-%{+YYYY.MM.dd}.txt":"8"}
- Odps:
configs:
redis.address: redis://:[email protected]:6379/1
redis.max.idle: 100
redis.max.total: 1024
redis.max.wait.mills: 3000
redis.timeout: 2000
redis.map.info.key: od-ps-cfg
redis.queue.info.key: od-ps-cfg-msg
http.map.info.api: http://172.16.1.52:81/api/v1/odps/provide_task_list/
task.thread.pool.size: 5000
#task.thread.cycle.commit.time: 30000
task.thread.commit.interval: 30000
task.tunnel.timezone: Asia/Shanghai
task.tunnel.retry.limit: 720 #commit日志失败的重试次数
task.tunnel.retry.interval: 5 #commit日志的间隔时间，秒级别
task.partitions.lru.size: 30000
task.report.status.address: 172.16.1.52:81
task.report.status.interval: 300000
scala.kafka.producer.brokerlist: 172.16.1.145:9092
scala.kafka.zookeeper: 172.16.1.181:2181,172.16.1.186:2181,172.16.1.226:2181/kafka
task.retry,kafka.groupid: odps_retry_event_group_test
task.retry.kafka.topic: odps_retry_event_topic_test
# - Elasticsearch5:
# hosts: ["172.16.1.145:9300"]
# indexTimezone: "UTC"
# cluster: poc_dtstack
# concurrentRequests: 1
# index: 'dtlog-%{tenant_id}-%{appname}-%{keeptype}-%{+YYYY.MM.dd}'
# errorEventLogConfig: {"/tmp/error-event-%{+YYYY.MM.dd}.txt":"3"}
# ERROR_PROTECT_KEYS: "@timestamp,appname,keeptype,logtype,tag,message,timestamp,local_ip,tenant_id,hostname,path,agent_type,offset,uuid,bajie_test"
# documentType: logs # default logs
# consistency: true # defalut false
# inputs:
# # - Beats:
# # codec: json
# # port: 8635
# - Kafka:
# codec: json
# encoding: UTF8 # defaut UTF8
# topic:
# dt_all_log: 5
# consumerSettings:
# group.id: dt_all_log_group
# zookeeper.connect: 172.16.1.181:2181,172.16.1.186:2181,172.16.1.226:2181/kafka
# auto.commit.interval.ms: "1000"
# filters:
# # - Performance:
# # path: /home/admin/jlogserver/logs/beat-filters-performance-%{+YYYY.MM.dd}.txt
# # timeZone: Asia/Shanghai
# # monitorPath: {"/home/admin/jlogserver/logs/beat-filters-performance-%{+YYYY.MM.dd}.txt":"8"}
# - dtstack.jdtloguserauth.DtLogUserAuth:
# apiServer: 172.16.1.52
# useSsl: false
# redisHost: 172.16.1.52
# redisPort: 6379
# isRedis: true
# redisDB: 1
# redisPassword: taukwnlwrd9
# - Add:
# fields: {"agent_type":"@metadata.beat","hostname":"beat.hostname","host":"beat.name"}
# - Remove:
# fields: ["@metadata","count","offset","beat"]
# - Rename:
# fields: {"source":"path"}
# - dtstack.jdtlogparser.DtLogParser:
# apiServer: 172.16.1.52:81
# useSsl: false
# redisHost: 172.16.1.52
# redisPort: 6379
# isRedis: true
# redisDB: 0
# redisPassword: taukwnlwrd9
# - dtstack.jdtlogcreatemessage.DtLogCreateMessage:
# repeatFields: ["path"]
# - Flow:
# configs:
# flow.control.counttype: unset
# flow.control.threshold: 10KB
# flow.stat.counttype: unset
# flow.stat.report.commit.delay.second: 3
# flow.stat.report.interval: 1000
# flow.stat.report.addr.template: "http://172.16.10.123:8854/api/logagent/test?uuid=%s&time=%s&bandwidth=%s"
# outputs:
# # - Performance:
# # path: /home/admin/jlogserver/logs/beat-performance-%{+YYYY.MM.dd}.txt
# # timeZone: Asia/Shanghai
# # monitorPath: {"/tmp/output-performance-%{+YYYY.MM.dd}.txt":"8"}
# - Netty:
# host: 172.16.1.58
# port: 8635
# openCompression: true
# compressionLevel: 6
# openCollectIp: true
# # format: ${HOSTNAME} ${appname} [${user_token} type=${logtype} tag=${logtag}] ${path} jlogstash/$${timestamp}/$${message}