Adding configurable status endpoint

This lets us match our internal health checks, I'm sure this will be
useful to others as well.
Not modifying Health() since it returns 200 of failure and there might
be people depending on the behavior.
Adding SimpleHealthCheck() that just validates DB connectivity, not
necessarily writes.
Adding config endpoints for status stuff

.gitignore

@@ -8,3 +8,4 @@ vagrant/db3-post-install.sh
 vagrant/db4-post-install.sh
 vagrant/vagrant-ssh-key
 vagrant/vagrant-ssh-key.pub
+Godeps/_workspace

go/config/config.go

@@ -108,6 +108,9 @@ type Configuration struct {
 	SSLCertFile                                string            // Name of SSL certification file, applies only when UseSSL = true
 	SSLCAFile                                  string            // Name of the Certificate Authority file, applies only when UseSSL = true
 	SSLValidOUs                                []string          // Valid organizational units when using mutual TLS
+	StatusEndpoint                             string            // Override the status endpoint.  Defaults to '/api/status'
+	StatusSimpleHealth                         bool              // If true, calling the status endpoint will use the simplified health check
+	StatusOUVerify                             bool              // If true, try to verify OUs when Mutual TLS is on.  Defaults to false
 	HttpTimeoutSeconds                         int               // Number of idle seconds before HTTP GET request times out (when accessing orchestrator-agent)
 	AgentPollMinutes                           uint              // Minutes between agent polling
 	UnseenAgentForgetHours                     uint              // Number of hours after which an unseen agent is forgotten
@@ -144,6 +147,9 @@ func NewConfiguration() *Configuration {
 		Debug:                                      false,
 		ListenAddress:                              ":3000",
 		AgentsServerPort:                           ":3001",
+		StatusEndpoint:                             "/api/status",
+		StatusSimpleHealth:                         true,
+		StatusOUVerify:                             false,
 		MySQLOrchestratorPort:                      3306,
 		MySQLTopologyMaxPoolConnections:            3,
 		MySQLTopologyUseMutualTLS:                  false,

go/http/api.go

@@ -19,13 +19,14 @@ package http
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/go-martini/martini"
-	"github.com/martini-contrib/auth"
-	"github.com/martini-contrib/render"
 	"net"
 	"net/http"
 	"strconv"
 
+	"github.com/go-martini/martini"
+	"github.com/martini-contrib/auth"
+	"github.com/martini-contrib/render"
+
 	"github.com/outbrain/orchestrator/go/agent"
 	"github.com/outbrain/orchestrator/go/config"
 	"github.com/outbrain/orchestrator/go/inst"
@@ -1705,6 +1706,27 @@ func (this *HttpAPI) LBCheck(params martini.Params, r render.Render, req *http.R
 	r.JSON(200, "OK")
 }
 
+// A configurable endpoint that can be for regular status checks or whatever.  While similar to
+// Health() this returns 500 on failure.  This will prevent issues for those that have come to
+// expect a 200
+// It might be a good idea to deprecate the current Health() behavior and roll this in at some
+// point
+func (this *HttpAPI) StatusCheck(params martini.Params, r render.Render, req *http.Request) {
+	// SimpleHealthTest just checks to see if we can connect to the database.  Lighter weight if you intend to call it a lot
+	var health *logic.HealthStatus
+	var err error
+	if config.Config.StatusSimpleHealth {
+		health, err = logic.SimpleHealthTest()
+	} else {
+		health, err = logic.HealthTest()
+	}
+	if err != nil {
+		r.JSON(500, &APIResponse{Code: ERROR, Message: fmt.Sprintf("Application node is unhealthy %+v", err), Details: health})
+		return
+	}
+	r.JSON(200, &APIResponse{Code: OK, Message: fmt.Sprintf("Application node is healthy"), Details: health})
+}
+
 // GrabElection forcibly grabs leadership. Use with care!!
 func (this *HttpAPI) GrabElection(params martini.Params, r render.Render, req *http.Request, user auth.User) {
 	if !isAuthorizedForAction(req, user) {
@@ -1999,4 +2021,6 @@ func (this *HttpAPI) RegisterRequests(m *martini.ClassicMartini) {
 	m.Get("/api/agent-seed-states/:seedId", this.AgentSeedStates)
 	m.Get("/api/agent-abort-seed/:seedId", this.AbortSeed)
 	m.Get("/api/seeds", this.Seeds)
+	// Configurable status check endpoint
+	m.Get(config.Config.StatusEndpoint, this.StatusCheck)
 }

go/logic/health_dao.go

@@ -18,6 +18,7 @@ package logic
 
 import (
 	"fmt"
+
 	"github.com/outbrain/golib/log"
 	"github.com/outbrain/golib/sqlutils"
 	"github.com/outbrain/orchestrator/go/db"
@@ -105,3 +106,22 @@ Cleanup:
 	}
 	return res, err
 }
+
+// Just check to make sure we can connect to the database
+func SimpleHealthTest() (*HealthStatus, error) {
+	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}
+
+	db, err := db.OpenOrchestrator()
+	if err != nil {
+		health.Error = err
+		return &health, log.Errore(err)
+	}
+
+	if err = db.Ping(); err != nil {
+		health.Error = err
+		return &health, log.Errore(err)
+	} else {
+		health.Healthy = true
+		return &health, nil
+	}
+}

go/ssl/ssl.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/go-martini/martini"
 	"github.com/outbrain/golib/log"
+	"github.com/outbrain/orchestrator/go/config"
 )
 
 // Determine if a string element is in a string array
@@ -53,8 +54,11 @@ func NewTLSConfig(caFile string, mutualTLS bool) (*tls.Config, error) {
 // Verify that the OU of the presented client certificate matches the list
 // of Valid OUs
 func Verify(r *nethttp.Request, validOUs []string) error {
+	if strings.Contains(r.URL.String(), config.Config.StatusEndpoint) && !config.Config.StatusOUVerify {
+		return nil
+	}
 	if r.TLS == nil {
-		return errors.New("no TLS")
+		return errors.New("No TLS")
 	}
 	for _, chain := range r.TLS.VerifiedChains {
 		s := chain[0].Subject.OrganizationalUnit

go/ssl/ssl_test.go

@@ -4,12 +4,14 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"io/ioutil"
 	nethttp "net/http"
 	"strings"
 	"syscall"
 	"testing"
 
+	"github.com/outbrain/orchestrator/go/config"
 	"github.com/outbrain/orchestrator/go/ssl"
 )
 
@@ -55,6 +57,24 @@ func TestNewTLSConfig(t *testing.T) {
 	}
 }
 
+func TestStatus(t *testing.T) {
+	var validOUs []string
+	url := fmt.Sprintf("http://example.com%s", config.Config.StatusEndpoint)
+
+	req, err := nethttp.NewRequest("GET", url, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	config.Config.StatusOUVerify = false
+	if err := ssl.Verify(req, validOUs); err != nil {
+		t.Errorf("Failed even with verification off")
+	}
+	config.Config.StatusOUVerify = true
+	if err := ssl.Verify(req, validOUs); err == nil {
+		t.Errorf("Did not fail on with bad verification")
+	}
+}
+
 func TestVerify(t *testing.T) {
 	var validOUs []string