-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
111 lines (88 loc) · 3.62 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
FROM centos:8
LABEL maintainer="Ilija Vukotic <[email protected]>"
RUN dnf update -y
RUN dnf list installed
RUN dnf install -y epel-release.noarch
RUN dnf update -y
# Install dependencies
RUN dnf install -y \
httpd \
git \
# curl \
wget \
rsync \
zip \
unzip \
vim \
libaio \
python3 \
# ntp \
krb5-workstation \
# mod_auth_kerb \
mod_ssl \
# mod_wsgi \
openssl-devel \
java-1.8.0-openjdk-devel
# mysql-connector-java
# build-essential \
# krb5-user \
# libzmq3-dev \
# pkg-config \
# software-properties-common \
# zlib1g-dev
# module-init-tools \
# openjdk-8-jdk \
# openjdk-8-jre-headless \
ENV JAVA_HOME /etc/alternatives/java_sdk_1.8.0_openjdk/jre/
# # es-hadoop - for all the modules (pig, mr, spark)
ENV EH_VERSION 7.12.0
RUN curl -LO https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-$EH_VERSION.zip
RUN unzip elasticsearch-hadoop-$EH_VERSION.zip && \
rm elasticsearch-hadoop-$EH_VERSION.zip
RUN mkdir /elasticsearch-hadoop && \
ln -s /elasticsearch-hadoop-$EH_VERSION/dist/elasticsearch-hadoop-$EH_VERSION.jar /elasticsearch-hadoop/elasticsearch-hadoop.jar && \
ln -s /elasticsearch-hadoop-$EH_VERSION/dist/elasticsearch-hadoop-pig-$EH_VERSION.jar /elasticsearch-hadoop/elasticsearch-hadoop-pig.jar && \
ln -s /elasticsearch-hadoop-$EH_VERSION/dist/elasticsearch-spark-20_2.11-$EH_VERSION.jar /elasticsearch-hadoop/elasticsearch-spark.jar
# hdfs - now needs subsription
# RUN wget http://archive.cloudera.com/cdh5/one-click-install/redhat/7/x86_64/cloudera-cdh-5-0.x86_64.rpm
# RUN yum install -y localinstall cloudera-cdh-5-0.x86_64.rpm
# RUN yum install -y pig \
# hbase \
# flume-ng
# # elephant-bird
# ENV EB_VERSION 4.17
# RUN wget http://central.maven.org/maven2/com/twitter/elephantbird/elephant-bird-hadoop-compat/$EB_VERSION/elephant-bird-hadoop-compat-$EB_VERSION.jar -O /usr/lib/pig/lib/elephant-bird-hadoop-compat.jar
# RUN wget http://central.maven.org/maven2/com/twitter/elephantbird/elephant-bird-core/$EB_VERSION/elephant-bird-core-$EB_VERSION.jar -O /usr/lib/pig/lib/elephant-bird-core.jar
# RUN wget http://central.maven.org/maven2/com/twitter/elephantbird/elephant-bird-pig/$EB_VERSION/elephant-bird-pig-$EB_VERSION.jar -O /usr/lib/pig/lib/elephant-bird-pig.jar
# mysql
RUN mkdir -p /usr/local/mysql && \
wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.13/mysql-connector-java-8.0.13.jar -O /usr/local/mysql/mysql-connector-java-8.0.13.jar
# RUN cp /usr/local/mysql/mysql-connector-java-5.1.45-bin.jar /usr/local/sqoop
ENV HADOOP_MAPRED_HOME /usr/lib/hadoop-mapreduce
ENV HADOOP_COMMON_HOME /usr/lib/hadoop
COPY configs/core-site.xml configs/hdfs-site.xml configs/mapred-site.xml configs/yarn-site.xml /etc/hadoop/conf/
COPY Unconfirmed.zip /usr/local/sqoop/lib/ojdbc6.jar
RUN chmod 755 /usr/local/sqoop/lib/ojdbc6.jar
# Oracle
RUN mkdir -p /opt/oracle/
COPY instantclient-basiclite-linux.x64-21.1.0.0.0.zip .
RUN unzip instantclient-basiclite-linux.x64-21.1.0.0.0.zip -d /opt/oracle/
RUN echo /opt/oracle/instantclient_21_1 > /etc/ld.so.conf.d/oracle-instantclient.conf
RUN python3 -m pip install --upgrade pip
RUN python3 -m pip install --no-cache-dir \
h5py \
tables \
numpy \
pandas \
scipy \
sklearn \
elasticsearch \
cx_Oracle \
requests \
mysql-connector \
stomp.py
COPY configs/krb5.conf /etc/krb5.conf
RUN useradd -ms /bin/bash analyticssvc
USER analyticssvc
WORKDIR /home/analyticssvc
CMD ["sleep", "99999999"]