SETUP

###
# Setup base VM to functioning VecNet VM
#
# date: 27.08.2014
# OS : Centos 6.5
#
###
# as root user
useradd dbrower
useradd blakshmi
useradd app
passwd dbrower
passwd blakshmi
passwd app
usermod -a -G wheel dbrower
usermod -a -G wheel blakshmi
usermod -a -G wheel app

# enable sudo access to wheel group
visudo
uncomment line:  %wheel  ALL=(ALL)       ALL

# now as normal user
wget http://mirrors.nebo.edu/public/epel/6/i386/epel-release-6-8.noarch.rpm
(Connection times out - replaced with https://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm)
sudo rpm -i epel-release-6-8.noarch.rpm
sudo yum update
sudo yum install htop
sudo yum install tmux
sudo yum install tree

# postgres
sudo yum install postgresql postgresql-devel
sudo yum install postgresql-server
sudo service postgresql initdb
sudo chkconfig postgresql on
sudo service postgresql start
sudo su - -c psql postgres
# at prompt type the following: << EOF
create user vecnet with password 'vecnet';
create database vecnet_qa;
grant all privileges on database vecnet_qa to vecnet;
alter database vecnet_qa owner to vecnet;
create database fedora;
create user fedora with password 'fedora';
grant all privileges on database fedora to fedora;
alter database fedora owner to fedora;
\q
EOF
sudo vim /var/lib/pgsql/data/pg_hba.conf
# change the end of the file to read << EOF
# "local" is for Unix domain socket connections only
local   all         postgres                          ident
local   all         all                               md5
# IPv4 local connections:
host    all         all         127.0.0.1/32          md5
# IPv6 local connections:
host    all         all         ::1/128               md5
EOF
sudo su - -c 'pg_ctl reload' postgres

# redis
sudo yum install redis
sudo chkconfig redis on
sudo service redis start
sudo iptables -I INPUT -p tcp --source 129.74.246.0/24 --dport 6379 -m state --state NEW -j ACCEPT
sudo service iptables save

# tomcat
sudo yum install tomcat6
sudo vim /etc/tomcat6/server.xml
# change line 69: 8080 -> 8081
sudo vim /etc/tomcat6/tomcat6.conf
# add line 22: JAVA_OPTS="-Xmx2G -Xms2G -Xmn512m"
# uncomment line 51, change 8080 -> 8081
sudo cp devops/etc/tomcat6/Catalina/localhost/* /etc/tomcat6/Catalina/localhost
(devops directory in root dir of repo)
sudo chkconfig tomcat6 on
sudo service tomcat6 start
# make hole in firewall (optional)
sudo iptables -I INPUT -p tcp --dport 8983 -s 10.0.0.0/8 -m state --state NEW -j ACCEPT
# for production: open hole for workers
sudo iptables -I INPUT -p tcp --dport 8081 --source 129.74.246.0/24 -m state --state NEW  -j ACCEPT
sudo /sbin/service iptables save

# fedora
sudo mkdir -p /opt/fedora
wget http://sourceforge.net/projects/fedora-commons/files/fedora/3.6.2/fcrepo-installer-3.6.2.jar
sudo java -jar fcrepo-installer-3.6.2.jar

# enter at the prompts:
# custom
# /opt/fedora
# fedoraAdmin
# [localhost] (use machine hostname??)
# [fedora]
# [false] (authentication for apia)
# false (ssl)
# existingTomcat
# /usr/share/tomcat6
# 8081 (tomcat port)
# [8005] default
# postgresql (database to use)

# here got this prompt instead :

Postgresql JDBC driver
----------------------
You may either use the included JDBC driver or your own copy.
Enter 'included' to use the included JDBC driver, or, enter the location
(full path) of the driver.


Enter a value [default is included] ==>)

# so the answer should be : # [included] (jdbc driver)


# fedora (user)
# fedora (pass)
# jbdc:postgresql://localhost/fedora   (jdbc url)
# [org.postgresql.Driver] (driver class)
# [false] (use upstream http auth)
# [false] (enable fesl)
# [true] (xacml enabled)
# [akubra-fs] (low level storage)
# [false] (enable RI)
# [false] (enable messaging)
# [false] (deploy local services)
##### XXX: this overwrote the /etc/tomcat6/Catalina/localhost/fedora.xml file
sudo chown tomcat:tomcat -R /opt/fedora

# solr
sudo yum install log4j
wget https://archive.apache.org/dist/lucene/solr/4.3.0/solr-4.3.0.tgz
tar xvf solr-4.3.0.tgz

# this step doesn't quite do it.
#
#sudo cp solr-4.3.0/dist/ /opt/solr-4.3.0
sudo cp -r solr-4.3.0/example/solr /opt/solr-4.3.0
sudo mv /opt/solr-4.3.0/collection1 /opt/solr-4.3.0/vecnet
sudo cp /opt/solr-4.3.0/vecnet/conf/stopwords.txt /opt/solr-4.3.0/vecnet/conf/stopwords_en.txt
sudo cp solr-4.3.0/dist/solr-4.3.0.war /opt/solr-4.3.0
sudo mkdir /opt/solr-4.3.0/lib
sudo cp solr-4.3.0/dist/*.jar /opt/solr-4.3.0/lib
sudo cp solr-4.3.0/contrib /opt/solr-4.3.0/lib
sudo cp solr-4.3.0/contrib/analysis-extras/lib/*.jar /opt/solr-4.3.0/lib
sudo cp solr-4.3.0/contrib/analysis-extras/lucine-libs/*.jar /opt/solr-4.3.0/lib
sudo cp -r solr-4.3.0/example/lib/ext/* /usr/share/tomcat6/lib
sudo chown tomcat:tomcat -R /opt/solr-4.3.0
# edit /etc/tomcat6/catalina.properties
# line 47 to read
# common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,/opt/solr-4.3.0/lib/ext/*.jar,/opt/solr-4.3.0/lib/contrib/extraction/lib/*.jar

# fits
wget http://projects.iq.harvard.edu/files/fits/files/fits-0.8.0.zip
unzip fits-0.8.0.zip
sudo mv fits-0.8.0 /opt
sudo chown app:app -R /opt/fits-0.8.0/
sudo chmod +x /opt/fits-0.8.0/fits.sh
# in the file /opt/fits-0.8.0/xml/jhove/jhove.conf
# comment out the edu.harvard.hul.ois.jhove.module.XmlModule module (lines 35--37)


# ruby
# trying chruby as the ruby manager....
git clone https://github.com/sstephenson/ruby-build.git
cd ruby-build
sudo ./install.sh
sudo mkdir -p /opt/rubies
wget -O chruby-0.3.4.tar.gz https://github.com/postmodern/chruby/archive/v0.3.4.tar.gz
tar -xzvf chruby-0.3.4.tar.gz
cd chruby-0.3.4/
sudo make install
sudo /usr/local/bin/ruby-build 2.0.0-p353 /opt/rubies/2.0.0-p353

(may run into dependency errors here - needed : sudo yum install gcc-c++ glibc-headers glibc-devel openssl-devel readline libyaml-devel readline-devel zlib zlib-devel)

# execute next command as root
cp devops/etc/profile.d/chruby.sh /etc/profile.d

# nginx
sudo cp devops/etc/yum.repos.d/nginx.repo /etc/yum.repos.d
sudo yum install nginx
sudo usermod -a -G app nginx
sudo chmod g+r /home/app
# since .erb files are template files, may need to fiddle with file after copy
# to remove all "<% %>" tags
# (replaced <% domain %> tags with <current VM IP> = e.g.(130.56.248.39) )
sudo cp devops/etc/nginx/conf.d/vecnet.conf.erb /etc/nginx/conf.d
sudo rm /etc/nginx/conf.d/default.conf
sudo chown :app /etc/nginx/nginx.conf /etc/nginx/conf.d
sudo chmod g+w /etc/nginx/nginx.conf /etc/nginx/conf.d
sudo chkconfig nginx on
sudo service nginx start

# image magick
sudo yum install ImageMagick ImageMagick-devel

# logrotate
# run as root
sudo cp devops/etc/logrotate.d/vecnet /etc/logrotate.d

# init.d
# run as root
sudo cp devops/etc/init.d/* /etc/init.d
sudo chkconfig --add vecnet-server
sudo chkconfig --add vecnet-worker-pool
sudo chkconfig vecnet-server on
sudo chkconfig vecnet-worker-pool on

# setup iptables
sudo iptables -I INPUT -p tcp --dport 80 -j ACCEPT
sudo iptables -I INPUT -p tcp --dport 443 -j ACCEPT
sudo /sbin/service iptables save

# clamav
sudo yum install clamav clamav-devel clamav-db

# bibutils >= 5.0
wget http://sourceforge.net/projects/bibutils/files/bibutils_5.0_src.tgz
tar xvfz bibutils_5.0_src.tgz
cd bibutils_5.0
./configure && make
sudo make install

# setup application
#Copy public key into app user .ssh folder of whoever is going to deploy to that machine
#Run cap ENV deploy:setup in dev machine

#(e.g. added

#JCU attempt at Capistrano

desc "Setup for the JCU QA environment"
task :jcu_qa do
  common_setup

  set :rails_env,   'qa'
  set :domain,      'dl-dev.vecnet.org'

  server "130.56.248.39", :app, :web, :db, :work, :primary => true
end

to config/deploy.rb above the other tasks)

# (then ran cap jcu_qa setup:deploy)

#Install bundler manually in that host("primary" VM) machine
 chruby 2.0.0-p353
 gem install bundler
 git clone git://github.com/banurekha/vecnet.git
 (git clone git://github.com/vecnet/vecnet-dl.git)
(had to move capistrano 'vecnet' dir temporarily to allow git to clone)
 cd vecnet
 bundle install
git clone git://github.com/vecnet/vecnet-dl.git
(au-m.rubygems.org timed out for solrizer-fedora.gem so wget bb-m.rubygems to get the .gem file and gem install first.)

#Setup worker Machine
Follow all the setup procedure described about
You could skip Fedora, tomcat, solr, postgres sql,
Follow below step to openup postgres in production so that worker machine can talk to postgres (Following changes are in host machine)
    #Change iptable to open postgres port
    sudo iptables -I INPUT -p tcp --source 129.74.246.0/24 --dport 5432 -m state --state NEW  -j ACCEPT
    sudo /sbin/service iptables save
    #Change postgres setting  at
    /var/lib/pgsql/data/postgresql.conf to bind to '*'
    /var/lib/pgsql/data/pg_hba.conf to allow user access from remote hosts
    restart postgres service

# ==== Additional setup for golang processes ====
# install runit
git clone https://github.com/imeyer/runit-rpm
sudo yum install rpm-build rpmdevtools glibc-static
cd runit-rpm && ./build.sh
sudo yum install ~/rpmbuild/RPMS/x86_64/runit-2.1.1-6.el6.x86_64.rpm
sudo chown :app /etc/service
sudo chmod g+w /etc/service

# allow limited sudo access by app
visudo
add line: app ALL=NOPASSWD:service, sv

# install golang
sudo yum install golang hg
mkdir ~/gocode
export GOPATH=~/gocode
go get github.com/dbrower/disadis

# make service directory
mkdir -p ~/sv/disadis
(copy in the run and settings files)
sudo ln -s ~app/sv/disadis /etc/service/disadis

# install noids
go get github.com/ndlib/noids
sudo mkdir -p /opt/noids/pools
sudo chown app:app -R /opt/noids
mkdir -p ~/sv/noids
(copy in the run and settings files)
sudo ln -s ~app/sv/noids /etc/service/noids
curl localhost:13001/pools -F name=vecnet -F template=.reeddeeddk
# do this next step if there is already a fedora repo containing minted ids.
# otherwise, feel free to ignore.
curl localhost:13001/pools/vecnet/advancePast -F id=(determine id using method in noids repo)
# for production machine only, make hole in fw for noids server
sudo iptables -I INPUT -p tcp --dport 13001 --source 129.74.246.0/24 -m state --state NEW  -j ACCEPT
sudo /sbin/service iptables save