DevKit documentation and polishing (#786)

* DevKit documentation and polishing

* Lightweight Vagrantfile

* Marathon-related roles reorganization

* One more thing - tax rule

* Various fixes during tests

* Format

* Fix override

Vagrantfile

@@ -5,70 +5,12 @@ require 'fileutils'
 
 CONFIG = File.join(File.dirname(__FILE__), "vagrant.local.rb")
 
-$vb_memory = 1024*8
-$vb_cpu = 4
-$nginx_ip = "192.168.10.111"
-$local = true
+$nginx_ip = "$(hostname -i)"
 user = "vagrant"
 
 require CONFIG if File.readable?(CONFIG)
 
-def expose_ports(config)
-    # Mesos
-    config.vm.network :forwarded_port, guest: 5050, host: 5050, auto_correct: true
-
-    # Marathon
-    config.vm.network :forwarded_port, guest: 8080, host: 8080, auto_correct: true
-
-    # Kafka
-    config.vm.network :forwarded_port, guest: 9092, host: 9092, auto_correct: true
-
-    # Zookeeper
-    config.vm.network :forwarded_port, guest: 2181, host: 2181, auto_correct: true
-
-    # Schema Registry
-    config.vm.network :forwarded_port, guest: 8081, host: 8081, auto_correct: true
-
-    # PostgreSQL
-    config.vm.network :forwarded_port, guest: 5432, host: 5432, auto_correct: true
-
-    # Phoenix
-    config.vm.network :forwarded_port, guest: 9090, host: 9090, auto_correct: true
-
-    # ES
-    config.vm.network :forwarded_port, guest: 9200, host: 9200, auto_correct: true
-
-    # Kibana
-    config.vm.network :forwarded_port, guest: 5601, host: 5601, auto_correct: true
-
-    # Consul
-    config.vm.network :forwarded_port, guest: 8500, host: 8500, auto_correct: true
-
-    # Middlewarehouse
-    config.vm.network :forwarded_port, guest: 9292, host: 9292, auto_correct: true
-
-    #Nginx
-    config.vm.network :forwarded_port, guest: 80, host: 80, auto_correct: true
-
-    #Nginx https
-    config.vm.network :forwarded_port, guest: 443, host: 443, auto_correct: true
-end
-
 def tune_vm(config, opts = {})
-  cpus = opts[:cpus]
-  memory = opts[:memory]
-
-  config.vm.provider :virtualbox do |vb|
-    user = "vagrant"
-    vb.cpus = cpus if cpus
-    vb.memory = memory if memory
-  end
-
-  config.vm.provider :vmware_fusion do |v, override|
-    v.vmx["memsize"] = memory if memory
-    v.vmx["numvcpus"] = cpus if cpus
-  end
-
   config.vm.provider :google do |g, override|
     user = "ubuntu"
 
@@ -100,6 +42,7 @@ def tune_vm(config, opts = {})
     aws.access_key_id = ENV['AWS_ACCESS_KEY_ID']
     aws.secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
     aws.keypair_name = ENV['AWS_KEY_NAME']
+
     aws.associate_public_ip = true
     aws.elastic_ip = "52.38.159.101"
     aws.block_device_mapping = [{ 'DeviceName' => '/dev/sda1', 'Ebs.VolumeSize' => 100 }]
@@ -114,100 +57,44 @@ def tune_vm(config, opts = {})
 end
 
 Vagrant.configure("2") do |config|
-  $master = "master"
   user = ENV['GOOGLE_SSH_USERNAME'] || "vagrant"
 
-  tune_vm(config, cpus: $vb_cpu, memory: $vb_memory)
+  tune_vm(config)
 
   config.vm.define :appliance, primary: true do |app|
     app.vm.box = "base_appliance_16.04_20161129"
     app.vm.box_url = "https://s3.amazonaws.com/fc-dev-boxes/base_appliance_16.04_20161129.box"
-
     app.vm.network :private_network, ip: $nginx_ip
-    expose_ports(app)
-
-    # Workaround for mitchellh/vagrant#1867
-    if ARGV[1] and \
-       (ARGV[1].split('=')[0] == "--provider" or ARGV[2])
-      provider = (ARGV[1].split('=')[1] || ARGV[2]).chomp
-    else
-      provider = (ENV['VAGRANT_DEFAULT_PROVIDER'] || "virtualbox").chomp
-    end
-
-    if provider == "google"
-      puts 'Overriding Google-specific variables'
-      $nginx_ip = "`hostname -I | awk '{print $1}'`"
-      $local = false
-    end
 
     app.vm.provision "ansible" do |ansible|
-
       ansible.verbose = "v"
       ansible.playbook = "prov-shit/ansible/vagrant_appliance.yml"
       ansible.extra_vars = {
         user: user,
-        appliance_hostname: $nginx_ip,
         mesos_ip: $nginx_ip,
-        local_vagrant: $local,
         first_run: true,
         docker_tags: {
-            ashes: ENV['DOCKER_TAG_ASHES'] || $master,
-            firebrand: ENV['DOCKER_TAG_FIREBRAND'] || $master,
-            phoenix: ENV['DOCKER_TAG_PHOENIX'] || $master,
-            greenriver: ENV['DOCKER_TAG_GREENRIVER'] || $master,
-            middlewarehouse: ENV['DOCKER_TAG_MIDDLEWAREHOUSE'] || $master,
-            messaging: ENV['DOCKER_TAG_MESSAGING'] || $master,
-            isaac: ENV['DOCKER_TAG_ISAAC'] || $master,
-            solomon: ENV['DOCKER_TAG_SOLOMON'] || $master,
-            capture_consumer: ENV['DOCKER_TAG_CAPTURE_CONSUMER'] || $master,
-            gift_card_consumer: ENV['DOCKER_TAG_GIFT_CARD_CONSUMER'] || $master,
-            shipments_consumer: ENV['DOCKER_TAG_SHIPMENTS_CONSUMER'] || $master,
-            shipstation_consumer: ENV['DOCKER_TAG_SHIPSTATION_CONSUMER'] || $master,
-            stock_items_consumer: ENV['DOCKER_TAG_STOCK_ITEMS_CONSUMER'] || $master,
-            storefront_topdrawer: ENV['DOCKER_TAG_STOREFRONT_TOPDRAWER'] || $master,
-            storefront_tpg: ENV['DOCKER_TAG_STOREFRONT_TPG'] || $master,
-            marketplace: ENV['DOCKER_TAG_MARKETPLACE'] || $master,
-            marketplace_ui: ENV['DOCKER_TAG_MARKETPLACE_UI'] || $master,
-            product_search: ENV['DOCKER_TAG_PRODUCT_SEARCH'] || $master,
-            demo_search: ENV['DOCKER_TAG_DEMO_SEARCH'] || $master
+            ashes: ENV['DOCKER_TAG_ASHES'] || "master",
+            firebrand: ENV['DOCKER_TAG_FIREBRAND'] || "master",
+            phoenix: ENV['DOCKER_TAG_PHOENIX'] || "master",
+            greenriver: ENV['DOCKER_TAG_GREENRIVER'] || "master",
+            middlewarehouse: ENV['DOCKER_TAG_MIDDLEWAREHOUSE'] || "master",
+            messaging: ENV['DOCKER_TAG_MESSAGING'] || "master",
+            isaac: ENV['DOCKER_TAG_ISAAC'] || "master",
+            solomon: ENV['DOCKER_TAG_SOLOMON'] || "master",
+            capture_consumer: ENV['DOCKER_TAG_CAPTURE_CONSUMER'] || "master",
+            gift_card_consumer: ENV['DOCKER_TAG_GIFT_CARD_CONSUMER'] || "master",
+            shipments_consumer: ENV['DOCKER_TAG_SHIPMENTS_CONSUMER'] || "master",
+            shipstation_consumer: ENV['DOCKER_TAG_SHIPSTATION_CONSUMER'] || "master",
+            stock_items_consumer: ENV['DOCKER_TAG_STOCK_ITEMS_CONSUMER'] || "master",
+            storefront_topdrawer: ENV['DOCKER_TAG_STOREFRONT_TOPDRAWER'] || "master",
+            storefront_tpg: ENV['DOCKER_TAG_STOREFRONT_TPG'] || "master",
+            marketplace: ENV['DOCKER_TAG_MARKETPLACE'] || "master",
+            marketplace_ui: ENV['DOCKER_TAG_MARKETPLACE_UI'] || "master",
+            product_search: ENV['DOCKER_TAG_PRODUCT_SEARCH'] || "master",
+            demo_search: ENV['DOCKER_TAG_DEMO_SEARCH'] || "master"
         }
       }
     end
   end
-
-  config.vm.define :build, autostart: false do |app|
-    app.vm.box = "build16.04"
-    app.vm.box_url = "https://s3.amazonaws.com/fc-dev-boxes/build16.04.box"
-    app.vm.box_download_checksum = "550f65256533c6dd4bcb5278dfa46ffe"
-    app.vm.box_download_checksum_type = "md5"
-  end
-
-  config.vm.define :appliance_base, autostart: false do |app|
-    app.vm.box = "boxcutter/ubuntu1604"
-    app.vm.network :private_network, ip: $nginx_ip
-
-    app.vm.provision "shell", inline: "apt-get install -y python-minimal"
-    app.vm.provision "ansible" do |ansible|
-      ansible.verbose = "v"
-      ansible.playbook = "prov-shit/ansible/vagrant_appliance_base.yml"
-      ansible.extra_vars = {
-        user: user
-      }
-    end
-  end
-
-  config.vm.define :build_base, autostart: false do |app|
-    app.vm.box = "boxcutter/ubuntu1604"
-    app.vm.network :private_network, ip: $nginx_ip
-
-    app.vm.provision "shell", inline: "apt-get install -y python-minimal"
-    app.vm.provision "ansible" do |ansible|
-      ansible.verbose = "v"
-      ansible.skip_tags = "buildkite"
-      ansible.playbook = "prov-shit/ansible/vagrant_builder.yml"
-      ansible.extra_vars = {
-        user: user
-      }
-    end
-  end
 end

engineering-wiki/devops/Development-Kit.md

@@ -0,0 +1,84 @@
+# Development Kit
+
+Each developer appliance has a batteries included for comfortable development.
+
+Navigation:
+
+* [Core software](#core-software)
+* [Programming languages](#programming-languages)
+* [Build tools](#build-tools)
+* [Command line utilities](#command-line-utilities)
+
+## Core software
+
+### Tier I
+
+Literally a spine of our platform, list of open-source applications we depend on:
+
+* [Apache Mesos](http://mesos.apache.org) - makes our datacenters multi-tenant, allowing multiple applications to work on the same machine.
+    * [Marathon](https://mesosphere.github.io/marathon) - container orchestration platform for Mesos.
+    * [mesos-consul](https://github.com/CiscoCloud/mesos-consul) - Mesos to Consul bridge for service discovery.
+* [Confluent Platform](https://www.confluent.io/product) - complete streaming platform for large-scale distributed environments.
+    * [Apache Kafka](https://kafka.apache.org) - a distributed streaming platform.
+    * [Apache Zookeeper](https://zookeeper.apache.org) - centralized service for providing distributed services synchronization.
+    * [Schema Registry](https://github.com/confluentinc/schema-registry) - provides a serving layer for metadata in Kafka.
+* [Consul](https://www.consul.io) - highly available and distributed service discovery.
+    * [consul-template](https://github.com/hashicorp/consul-template) - generic template rendering and notifications with Consul.
+* [Docker](https://www.docker.com) - software containerization platform.
+* [Elasticsearch](https://www.elastic.co/products/elasticsearch) - distributed, RESTful search and analytics engine.
+    * [Kibana](https://www.elastic.co/products/kibana) - data visualization system for Elasticsearch.
+* [Nginx](https://www.nginx.com) - high performance load balancer, web server and reverse proxy.
+* [PostgreSQL](https://www.postgresql.org) - open source object-relational database system.
+    * [bottledwater-pg](https://github.com/confluentinc/bottledwater-pg) - data capture system from PostgreSQL into Kafka.
+    * [pgweb](https://github.com/sosedoff/pgweb) - web-based database browser for PostgreSQL.
+* [rsyslog](http://www.rsyslog.com) - open-source software log-forwarding system.
+
+### Tier II
+
+Available only on production environments or separate instances:
+
+* [Docker Registry](https://docs.docker.com/registry) - Docker images storage and distribution system.
+* [OpenVPN](https://openvpn.net/index.php/open-source.html) - open source SSL VPN solution.
+* [Sinopia](https://github.com/rlidwka/sinopia) - private NPM repository server.
+* [consul-alerts](https://github.com/AcalephStorage/consul-alerts) - a simple daemon to send notifications based on Consul health checks.
+* [marathon-alerts](https://github.com/ashwanthkumar/marathon-alerts) - tool for monitoring the apps running on Marathon.
+
+## Programming languages
+
+Pre-installed compilers and runtimes, in case you need to build or test something inside a VM:
+
+* Oracle JVM 1.8
+* [Go](https://golang.org) 1.7.4
+* [Node.js](https://nodejs.org) 7.1.0
+* [Elixir](http://elixir-lang.org) 1.1.0
+
+## Build tools
+
+Various package management and build systems which are necessary for modern development:
+
+* Scala
+    * [sbt](http://www.scala-sbt.org) - interactive build tool for Scala.
+* Go
+    * [glide](https://github.com/Masterminds/glide) - package management for Golang.
+* Clojure
+    * [lein](https://github.com/technomancy/leiningen) - automating Clojure projects without setting your hair on fire.
+    * [boot](https://github.com/boot-clj/boot) - Clojure build framework and ad-hoc Clojure script evaluator.
+* JavaScript
+    * [babel-cli](https://github.com/babel/babel) - command-line compiler for writing next generation JavaScript.
+    * [flow-bin](https://github.com/flowtype/flow-bin) - Binary wrapper for Flow - a static type checker for JavaScript.
+    * [gulp](https://github.com/gulpjs/gulp) - streaming build system for Node.js.
+    * [yarn](https://yarnpkg.com) - fast, reliable, and secure dependency management for Node.js.
+
+## Command line utilities
+
+Baked in command-line utilities which ease problem solving:
+
+* [consulate](https://github.com/gmr/consulate) - command-line client for the Consul HTTP API.
+* [docker-clean](https://github.com/ZZROTDesign/docker-clean) - a simple shell script to clean up the Docker Daemon.
+* [docker-compose](https://github.com/docker/compose) - define and run multi-container applications with Docker.
+* [httpie](https://httpie.org) - a command line HTTP client that will make you smile.
+* [jq](https://stedolan.github.io/jq) - lightweight and flexible command-line JSON processor.
+* [kt](https://github.com/fgeller/kt) - commandline tool for Apache Kafka.
+* [ncdu](https://dev.yorhel.nl/ncdu) - disk usage analyzer with an ncurses interface.
+* [pgcli](https://github.com/dbcli/pgcli) - Postgres CLI with autocompletion and syntax highlighting.
+* [zookeepercli](https://github.com/outbrain/zookeepercli) - simple, lightweight, dependable CLI for Zookeeper.

engineering-wiki/devops/README.md

@@ -3,8 +3,8 @@
 Contents:
 
 * [Deploying Custom Branches](Deploying-Custom-Branches.md)
+* [Development-Kit](Development-Kit.md)
 * [Git Hooks](Git-Hooks.md)
-* [Local VM Development Environment](Local-VM-DevEnv.md)
 
 External:
 

prov-shit/ansible/bootstrap_kangaroos.yml

@@ -23,17 +23,9 @@
     with_gatling_seeder: true
     with_mwh_seeder: true
   roles:
-    - { role: dev/phoenix }
-    - { role: dev/greenriver }
-    - { role: dev/ashes }
-    - { role: dev/firebrand }
-    - { role: dev/middlewarehouse }
-    - { role: dev/messaging }
-    - { role: dev/isaac }
-    - { role: dev/solomon }
+    - { role: dev/marathon }
     - { role: dev/balancer }
     - { role: demo/balancer }
-    - { role: dev/consumers }
     - { role: dev/seeder }
 
   handlers: