-
Notifications
You must be signed in to change notification settings - Fork 1
183 lines (183 loc) · 16.9 KB
/
[email protected]_20201213_214408.rtf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
{\rtf1\ansi\ansicpg1251\deff0\deflang1049{\fonttbl{\f0\fmodern Consolas;}{\f1\fmodern\fcharset204 Consolas;}{\f2\fnil\fcharset129 Courier New;}}
{\colortbl ;\red20\green20\blue20;\red142\green142\blue142;\red16\green158\blue98;\red148\green146\blue12;\red170\green51\blue170;\red170\green64\blue64;\red56\green136\blue159;\red85\green85\blue85;}
\viewkind4\uc1\pard\f0\fs20 Authenticating with public key "imported-openssh-key"\cf1\highlight2
\par \cf0\highlight0 \u9484?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9488?\cf1\highlight2
\par \cf0\highlight0 \u9474? \cf3\f1\bullet MobaXterm 20.6 \bullet \cf0\f0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \cf4 (SSH client, X-server and networking tools)\cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u8594? SSH session to \cf5 BD_274_mmingalov\cf0 @\cf5 89.208.223.141 \cf0\u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet SSH compression : \cf3\f0 v\cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet SSH-browser : \cf3\f0 v\cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet X11-forwarding : \cf6\f0 x\cf0 (disabled or not supported by server) \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet DISPLAY : \cf5\f0 192.168.1.44:0.0 \cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u8594? For more info, ctrl+click on \cf7\ul help\cf0\ulnone or visit our \cf7\ul website\cf0\ulnone \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9492?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9496?\cf1\highlight2
\par
\par \cf0\highlight0 Last login: Sun Dec 13 10:52:23 2020 from host.91-235-148-103.ipoe.khd.ru\cf1\highlight2
\par \cf0\highlight0 [BD_274_mmingalov@bigdataanalytics-worker-0 ~]$ pyspark\cf1\highlight2
\par \cf0\highlight0 SPARK_MAJOR_VERSION is set to 2, using Spark2\cf1\highlight2
\par \cf0\highlight0 Python 2.7.5 (default, Apr 2 2020, 13:16:51)\cf1\highlight2
\par \cf0\highlight0 [GCC 4.8.5 20150623 (Red Hat 4.8.5-39)] on linux2\cf1\highlight2
\par \cf0\highlight0 Type "help", "copyright", "credits" or "license" for more information.\cf1\highlight2
\par \cf0\highlight0 Setting default log level to "WARN".\cf1\highlight2
\par \cf0\highlight0 To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLeve l(newLevel).\cf1\highlight2
\par \cf0\highlight0 Welcome to\cf1\highlight2
\par \cf0\highlight0 ____ __\cf1\highlight2
\par \cf0\highlight0 / __/__ ___ _____/ /__\cf1\highlight2
\par \cf0\highlight0 _\\ \\/ _ \\/ _ `/ __/ '_/\cf1\highlight2
\par \cf0\highlight0 /__ / .__/\\_,_/_/ /_/\\_\\ version 2.3.2.3.1.4.0-315\cf1\highlight2
\par \cf0\highlight0 /_/\cf1\highlight2
\par
\par \cf0\highlight0 Using Python version 2.7.5 (default, Apr 2 2020 13:16:51)\cf1\highlight2
\par \cf0\highlight0 SparkSession available as 'spark'.\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf6\highlight0 Network error: Software caused connection abort\cf1\highlight2
\par
\par \cf8\highlight0\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\cf1\highlight2
\par
\par \cf6\highlight0 Session stopped\cf1\highlight2
\par \cf0\highlight0 - Press \cf5 <return>\cf0 to exit tab\cf1\highlight2
\par \cf0\highlight0 - Press \cf5 R\cf0 to restart session\cf1\highlight2
\par \cf0\highlight0 - Press \cf5 S\cf0 to save terminal output to file\cf1\highlight2
\par \cf0\highlight0 Authenticating with public key "imported-openssh-key"\cf1\highlight2
\par \cf0\highlight0 \u9484?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9488?\cf1\highlight2
\par \cf0\highlight0 \u9474? \cf3\f1\bullet MobaXterm 20.6 \bullet \cf0\f0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \cf4 (SSH client, X-server and networking tools)\cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u8594? SSH session to \cf5 BD_274_mmingalov\cf0 @\cf5 89.208.223.141 \cf0\u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet SSH compression : \cf3\f0 v\cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet SSH-browser : \cf3\f0 v\cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet X11-forwarding : \cf6\f0 x\cf0 (disabled or not supported by server) \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474?\f1 \bullet DISPLAY : \cf5\f0 192.168.1.44:0.0 \cf0 \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9474? \u8594? For more info, ctrl+click on \cf7\ul help\cf0\ulnone or visit our \cf7\ul website\cf0\ulnone \u9474?\cf1\highlight2
\par \cf0\highlight0 \u9492?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9472?\u9496?\cf1\highlight2
\par
\par \cf0\highlight0 Last login: Sun Dec 13 10:58:58 2020 from host.91-235-148-103.ipoe.khd.ru\cf1\highlight2
\par \cf0\highlight0 [BD_274_mmingalov@bigdataanalytics-worker-0 ~]$ from pyspark.sql import SparkSession\cf1\highlight2
\par \cf0\highlight0 -bash: from: command not found\cf1\highlight2
\par \cf0\highlight0 [BD_274_mmingalov@bigdataanalytics-worker-0 ~]$ from pyspark.sql import functions as F\cf1\highlight2
\par \cf0\highlight0 -bash: from: command not found\cf1\highlight2
\par \cf0\highlight0 [BD_274_mmingalov@bigdataanalytics-worker-0 ~]$ from pyspark.sql.types import StructType, StringType\cf1\highlight2
\par \cf0\highlight0 -bash: from: command not found\cf1\highlight2
\par \cf0\highlight0 [BD_274_mmingalov@bigdataanalytics-worker-0 ~]$\cf1\highlight2
\par \cf0\highlight0 [BD_274_mmingalov@bigdataanalytics-worker-0 ~]$\cf1\highlight2
\par \cf0\highlight0 [BD_274_mmingalov@bigdataanalytics-worker-0 ~]$ pyspark\cf1\highlight2
\par \cf0\highlight0 SPARK_MAJOR_VERSION is set to 2, using Spark2\cf1\highlight2
\par \cf0\highlight0 Python 2.7.5 (default, Apr 2 2020, 13:16:51)\cf1\highlight2
\par \cf0\highlight0 [GCC 4.8.5 20150623 (Red Hat 4.8.5-39)] on linux2\cf1\highlight2
\par \cf0\highlight0 Type "help", "copyright", "credits" or "license" for more information.\cf1\highlight2
\par \cf0\highlight0 Setting default log level to "WARN".\cf1\highlight2
\par \cf0\highlight0 To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\cf1\highlight2
\par \cf0\highlight0 20/12/13 11:18:05 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\cf1\highlight2
\par \cf0\highlight0 Welcome to\cf1\highlight2
\par \cf0\highlight0 ____ __\cf1\highlight2
\par \cf0\highlight0 / __/__ ___ _____/ /__\cf1\highlight2
\par \cf0\highlight0 _\\ \\/ _ \\/ _ `/ __/ '_/\cf1\highlight2
\par \cf0\highlight0 /__ / .__/\\_,_/_/ /_/\\_\\ version 2.3.2.3.1.4.0-315\cf1\highlight2
\par \cf0\highlight0 /_/\cf1\highlight2
\par
\par \cf0\highlight0 Using Python version 2.7.5 (default, Apr 2 2020 13:16:51)\cf1\highlight2
\par \cf0\highlight0 SparkSession available as 'spark'.\cf1\highlight2
\par \cf0\highlight0 >>> from pyspark.sql import SparkSession\cf1\highlight2
\par \cf0\highlight0 >>> from pyspark.sql import functions as F\cf1\highlight2
\par \cf0\highlight0 >>> from pyspark.sql.types import StructType, StringType\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>> def console_output(df, freq):\cf1\highlight2
\par \cf0\highlight0 ... return df.writeStream \\\cf1\highlight2
\par \cf0\highlight0 ... .format("console") \\\cf1\highlight2
\par \cf0\highlight0 ... .trigger(processingTime='%s seconds' % freq ) \\\cf1\highlight2
\par \cf0\highlight0 ... .options(truncate=False) \\\cf1\highlight2
\par \cf0\highlight0 ... .start()\cf1\highlight2
\par \cf0\highlight0 ...\cf1\highlight2
\par \cf0\highlight0 >>> schema = StructType() \\\cf1\highlight2
\par \cf0\highlight0 ... .add("LICENSE_NUMBER", StringType()) \\\cf1\highlight2
\par \cf0\highlight0 ... .add("MINE_SITE", StringType())\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>> raw_files = spark \\\cf1\highlight2
\par \cf0\highlight0 ... .readStream \\\cf1\highlight2
\par \cf0\highlight0 ... .format("csv") \\\cf1\highlight2
\par \cf0\highlight0 ... .schema(schema) \\\cf1\highlight2
\par \cf0\highlight0 ... .options(path="input_csv_for_stream", header=True) \\\cf1\highlight2
\par \cf0\highlight0 ... .load()\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>> out = console_output(raw_files, 5)\cf1\highlight2
\par \cf0\highlight0 -------------------------------------------\cf1\highlight2
\par \cf0\highlight0 Batch: 0\cf1\highlight2
\par \cf0\highlight0 -------------------------------------------\cf1\highlight2
\par \cf0\highlight0 +--------------+--------------------+\cf1\highlight2
\par \cf0\highlight0 |LICENSE_NUMBER|MINE_SITE |\cf1\highlight2
\par \cf0\highlight0 +--------------+--------------------+\cf1\highlight2
\par \cf0\highlight0 |GB1274 |Berezitoviy |\cf1\highlight2
\par \cf0\highlight0 |GM0366 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GM0367 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GM0368 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GM0369 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GB1282 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GB1420 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GB1463 |Polymetal Yakutsk |\cf1\highlight2
\par \cf0\highlight0 |GB1464 |Polymetal Yakutsk |\cf1\highlight2
\par \cf0\highlight0 |GB1540 |Polymetal Khabarovsk|\cf1\highlight2
\par \cf0\highlight0 |GB1275 |Tabornoe |\cf1\highlight2
\par \cf0\highlight0 |GB0901 |Seligdar |\cf1\highlight2
\par \cf0\highlight0 |GB1635 |Seligdar |\cf1\highlight2
\par \cf0\highlight0 |GM0323 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0324 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0325 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0326 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0327 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0328 |Alel |\cf1\highlight2
\par \cf0\highlight0 |GM0329 |Alel |\cf1\highlight2
\par \cf0\highlight0 +--------------+--------------------+\cf1\highlight2
\par \cf0\highlight0 only showing top 20 rows\cf1\highlight2
\par
\par \cf0\highlight0 20/12/13 11:33:36 WARN ProcessingTimeExecutor: Current batch is falling behind. The trigger interval is 5000 milliseconds, but spent 10440 milliseconds\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 Traceback (most recent call last):\cf1\highlight2
\par \cf0\highlight0 File "/usr/hdp/current/spark2-client/python/pyspark/context.py", line 261, in signal_handler\cf1\highlight2
\par \cf0\highlight0 raise KeyboardInterrupt()\cf1\highlight2
\par \cf0\highlight0 KeyboardInterrupt\cf1\highlight2
\par \cf0\highlight0 >>> out.stop()\cf1\highlight2
\par \cf0\highlight0 >>> out = console_output(raw_files, 15)\cf1\highlight2
\par \cf0\highlight0 -------------------------------------------\cf1\highlight2
\par \cf0\highlight0 Batch: 0\cf1\highlight2
\par \cf0\highlight0 -------------------------------------------\cf1\highlight2
\par \cf0\highlight0 +--------------+--------------------+\cf1\highlight2
\par \cf0\highlight0 |LICENSE_NUMBER|MINE_SITE |\cf1\highlight2
\par \cf0\highlight0 +--------------+--------------------+\cf1\highlight2
\par \cf0\highlight0 |GB1274 |Berezitoviy |\cf1\highlight2
\par \cf0\highlight0 |GM0366 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GM0367 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GM0368 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GM0369 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GB1282 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GB1420 |U1 |\cf1\highlight2
\par \cf0\highlight0 |GB1463 |Polymetal Yakutsk |\cf1\highlight2
\par \cf0\highlight0 |GB1464 |Polymetal Yakutsk |\cf1\highlight2
\par \cf0\highlight0 |GB1540 |Polymetal Khabarovsk|\cf1\highlight2
\par \cf0\highlight0 |GB1275 |Tabornoe |\cf1\highlight2
\par \cf0\highlight0 |GB0901 |Seligdar |\cf1\highlight2
\par \cf0\highlight0 |GB1635 |Seligdar |\cf1\highlight2
\par \cf0\highlight0 |GM0323 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0324 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0325 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0326 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0327 |Holbinskiy |\cf1\highlight2
\par \cf0\highlight0 |GM0328 |Alel |\cf1\highlight2
\par \cf0\highlight0 |GM0329 |Alel |\cf1\highlight2
\par \cf0\highlight0 +--------------+--------------------+\cf1\highlight2
\par \cf0\highlight0 only showing top 20 rows\cf1\highlight2
\par
\par
\par \cf0\highlight0 ...\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \cf0\highlight0 >>> out.stop()\cf1\highlight2
\par \cf0\highlight0 >>>\cf1\highlight2
\par \pard\cf0\highlight0\f2\fs18
\par }