diff --git a/phoenix-scala/sql/V5.20170619200953__create_channels_table.sql b/phoenix-scala/sql/V5.20170619200953__create_channels_table.sql new file mode 100644 index 0000000000..aeb39f3896 --- /dev/null +++ b/phoenix-scala/sql/V5.20170619200953__create_channels_table.sql @@ -0,0 +1,9 @@ +create table channels ( + id serial primary key, + intelligence_channel_id integer not null, + scope exts.ltree not null, + name generic_string not null, + purchase_location integer not null, + created_at generic_timestamp not null, + updated_at generic_timestamp not null +); diff --git a/phoenix-scala/sql/V5.20170619201616__create_channels_search_view.sql b/phoenix-scala/sql/V5.20170619201616__create_channels_search_view.sql new file mode 100644 index 0000000000..68a70f5f3c --- /dev/null +++ b/phoenix-scala/sql/V5.20170619201616__create_channels_search_view.sql @@ -0,0 +1,10 @@ +create table channels_search_view ( + id bigint primary key, + scope exts.ltree not null, + name generic_string not null, + hosts jsonb default '[]', + organization_name generic_string not null, + purchase_location generic_string not null, + created_at json_timestamp, + updated_at json_timestamp +); diff --git a/remote/Dockerfile b/remote/Dockerfile new file mode 100644 index 0000000000..7faf20340e --- /dev/null +++ b/remote/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:alpine + +RUN apk add --no-cache ca-certificates + +RUN mkdir -p /remote +ADD . /go/src/github.com/FoxComm/highlander/remote +WORKDIR /go/src/github.com/FoxComm/highlander/remote +RUN go build -o remote main.go && \ + cp remote /remote && \ + rm -rf /go +WORKDIR /remote + +CMD /remote/remote 2>&1 | tee /logs/remote.log diff --git a/remote/controllers/channels.go b/remote/controllers/channels.go index f05eec7c10..4c3733c60e 100644 --- a/remote/controllers/channels.go +++ b/remote/controllers/channels.go @@ -3,32 +3,28 @@ package controllers import ( "net/http" - "github.com/FoxComm/highlander/remote/models/phoenix" "github.com/FoxComm/highlander/remote/payloads" "github.com/FoxComm/highlander/remote/responses" "github.com/FoxComm/highlander/remote/services" "github.com/FoxComm/highlander/remote/utils/failures" - "github.com/jinzhu/gorm" ) type Channels struct { - phxDB *gorm.DB + dbs *services.RemoteDBs } -func NewChannels(phxDB *gorm.DB) *Channels { - return &Channels{phxDB: phxDB} +func NewChannels(dbs *services.RemoteDBs) *Channels { + return &Channels{dbs: dbs} } // GetChannel finds a single channel by its ID. func (ctrl *Channels) GetChannel(id int) ControllerFunc { return func() (*responses.Response, failures.Failure) { - channel := &phoenix.Channel{} - - if err := services.FindChannelByID(ctrl.phxDB, id, channel); err != nil { - return nil, err + resp, fail := services.FindChannelByID(ctrl.dbs, id) + if fail != nil { + return nil, fail } - resp := responses.NewChannel(channel) return responses.NewResponse(http.StatusOK, resp), nil } } @@ -36,13 +32,31 @@ func (ctrl *Channels) GetChannel(id int) ControllerFunc { // CreateChannel creates a new channel. func (ctrl *Channels) CreateChannel(payload *payloads.CreateChannel) ControllerFunc { return func() (*responses.Response, failures.Failure) { - phxChannel := payload.PhoenixModel() - - if err := services.InsertChannel(ctrl.phxDB, phxChannel); err != nil { - return nil, err + resp, fail := services.InsertChannel(ctrl.dbs, payload) + if fail != nil { + return nil, fail } - resp := responses.NewChannel(phxChannel) return responses.NewResponse(http.StatusCreated, resp), nil } } + +// UpdateChannel updates an existing channel. +func (ctrl *Channels) UpdateChannel(id int, payload *payloads.UpdateChannel) ControllerFunc { + return func() (*responses.Response, failures.Failure) { + + // existingPhxChannel := &phoenix.Channel{} + // if err := services.FindChannelByID(ctrl.dbs, id, existingPhxChannel); err != nil { + // return nil, err + // } + + // phxChannel := payload.PhoenixModel(existingPhxChannel) + // if err := services.UpdateChannel(ctrl.dbs, phxChannel); err != nil { + // return nil, err + // } + + // resp := responses.NewChannel(phxChannel) + // return responses.NewResponse(http.StatusOK, resp), nil + return nil, nil + } +} diff --git a/remote/controllers/controllers.go b/remote/controllers/controllers.go index 7605e7e2ac..ce74a9e7cd 100644 --- a/remote/controllers/controllers.go +++ b/remote/controllers/controllers.go @@ -15,16 +15,20 @@ func Start() { log.Fatal(err) } - phxDB, err := services.NewPhoenixConnection(config) + dbs, err := services.NewRemoteDBs(config) if err != nil { log.Fatal(err) } - defer phxDB.Close() - channelsCtrl := NewChannels(phxDB) + channelsCtrl := NewChannels(dbs) + pingCtrl := NewPing(dbs) r := NewRouter() + r.GET("/v1/public/health", func(fc *FoxContext) error { + return fc.Run(pingCtrl.GetHealth()) + }) + r.GET("/v1/public/channels/:id", func(fc *FoxContext) error { id := fc.ParamInt("id") return fc.Run(channelsCtrl.GetChannel(id)) @@ -36,5 +40,13 @@ func Start() { return fc.Run(channelsCtrl.CreateChannel(&payload)) }) + r.PATCH("/v1/public/channels/:id", func(fc *FoxContext) error { + id := fc.ParamInt("id") + payload := payloads.UpdateChannel{} + fc.BindJSON(&payload) + + return fc.Run(channelsCtrl.UpdateChannel(id, &payload)) + }) + r.Run(config.Port) } diff --git a/remote/controllers/fox_context.go b/remote/controllers/fox_context.go index a56cbf5749..2bfe4e7dfb 100644 --- a/remote/controllers/fox_context.go +++ b/remote/controllers/fox_context.go @@ -24,6 +24,37 @@ func NewFoxContext(c echo.Context) *FoxContext { return &FoxContext{c, nil} } +func (fc *FoxContext) getJWT() (*JWT, failures.Failure) { + jwtStr, fail := fc.getJWTString() + if fail != nil { + return nil, fail + } + + jwt, err := NewJWT(jwtStr) + if err != nil { + return nil, failures.New(err) + } + + return jwt, nil +} + +func (fc *FoxContext) getJWTString() (string, failures.Failure) { + // Try to get from the header first. + req := fc.Request() + jwt, ok := req.Header["Jwt"] + if ok && len(jwt) > 0 { + return jwt[0], nil + } + + // Try a cookie. + cookie, err := req.Cookie("JWT") + if err != nil { + return "", failures.New(err) + } + + return cookie.Value, nil +} + // BindJSON grabs the JSON payload and unmarshals it into the interface provided. func (fc *FoxContext) BindJSON(payload payloads.Payload) { if fc.failure != nil { @@ -39,6 +70,17 @@ func (fc *FoxContext) BindJSON(payload payloads.Payload) { fc.failure = err return } + + scoped, ok := payload.(payloads.Scoped) + if ok { + jwt, fail := fc.getJWT() + if fail != nil { + fc.failure = fail + return + } + + scoped.EnsureScope(jwt.Scope()) + } } // ParamInt parses an integer from the parameters list (as defined by the URI). diff --git a/remote/controllers/jwt.go b/remote/controllers/jwt.go new file mode 100644 index 0000000000..0adc9c816c --- /dev/null +++ b/remote/controllers/jwt.go @@ -0,0 +1,56 @@ +package controllers + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" +) + +// JWT is a simplified JWT wrapper for our usage. Since we rely on Isaac for +// validation, we just get customer and scope information. +type JWT struct { + Header map[string]interface{} + Payload map[string]interface{} +} + +// NewJWT parses the JWT from a string. +func NewJWT(jwtStr string) (*JWT, error) { + parts := strings.Split(jwtStr, ".") + if len(parts) != 3 { + return nil, errors.New("JWT is malformed") + } + + headerBytes, err := base64.URLEncoding.DecodeString(parts[0]) + if err != nil { + return nil, fmt.Errorf("Error decoding header with error: %s", err) + } + + header := map[string]interface{}{} + if err := json.Unmarshal(headerBytes, &header); err != nil { + return nil, fmt.Errorf("Error marshalling header with error: %s", err) + } + + payloadBytes, err := base64.URLEncoding.DecodeString(parts[1]) + if err != nil { + return nil, fmt.Errorf("Error decoding payload with error: %s", err) + } + + payload := map[string]interface{}{} + if err := json.Unmarshal(payloadBytes, &payload); err != nil { + return nil, fmt.Errorf("Error marshalling payload with error: %s", err) + } + + return &JWT{Header: header, Payload: payload}, nil +} + +// Scope gets the scope string passed in the JWT. +func (j JWT) Scope() string { + scope, ok := j.Payload["Scope"] + if !ok { + return "" + } + + return scope.(string) +} diff --git a/remote/controllers/ping.go b/remote/controllers/ping.go new file mode 100644 index 0000000000..9dd4012c69 --- /dev/null +++ b/remote/controllers/ping.go @@ -0,0 +1,50 @@ +package controllers + +import ( + "net/http" + + "github.com/FoxComm/highlander/remote/responses" + "github.com/FoxComm/highlander/remote/services" + "github.com/FoxComm/highlander/remote/utils/failures" +) + +// Ping is a really simple controller used for health checks. +type Ping struct { + dbs *services.RemoteDBs +} + +// NewPing creates a new ping controller. +func NewPing(dbs *services.RemoteDBs) *Ping { + return &Ping{dbs: dbs} +} + +type health struct { + Intelligence string `json:"intelligence"` + Phoenix string `json:"phoenix"` +} + +// GetHealth tests the connection to the databases and returns the status. +func (ctrl *Ping) GetHealth() ControllerFunc { + return func() (*responses.Response, failures.Failure) { + icPingErr := ctrl.dbs.IC().Ping() + phxPingErr := ctrl.dbs.Phx().Ping() + + statusCode := http.StatusOK + h := health{ + Intelligence: "passed", + Phoenix: "passed", + } + + if icPingErr != nil { + statusCode = http.StatusInternalServerError + h.Intelligence = "failed" + } + + if phxPingErr != nil { + statusCode = http.StatusInternalServerError + h.Phoenix = "failed" + } + + return responses.NewResponse(statusCode, h), nil + } +} diff --git a/remote/controllers/router.go b/remote/controllers/router.go index 957a4cace6..e04911ed26 100644 --- a/remote/controllers/router.go +++ b/remote/controllers/router.go @@ -42,3 +42,10 @@ func (r *Router) POST(uri string, rf RouterFunc) { return rf(fc) }) } + +func (r *Router) PATCH(uri string, rf RouterFunc) { + r.e.PATCH(uri, func(c echo.Context) error { + fc := c.(*FoxContext) + return rf(fc) + }) +} diff --git a/remote/glide.lock b/remote/glide.lock index c7ea5081ce..cba53557e9 100644 --- a/remote/glide.lock +++ b/remote/glide.lock @@ -1,12 +1,12 @@ hash: ecd68d1efcdac724b47d7da0a0c8fccf88a44ebf3888e32c6e1ab4eb8aef61ea -updated: 2017-06-16T17:47:29.132040807-07:00 +updated: 2017-06-19T20:46:05.130532456-07:00 imports: - name: github.com/jinzhu/gorm version: caa792644ce60fd7b429e0616afbdbccdf011be2 - name: github.com/jinzhu/inflection version: 74387dc39a75e970e7a3ae6a3386b5bd2e5c5cff - name: github.com/labstack/echo - version: 2d7cce467709a9e54ae3f8d77863e6d01859e134 + version: 7676f85ef9d4a50797e17b30c4b0b1cb3c7c532a - name: github.com/labstack/gommon version: 1121fd3e243c202482226a7afe4dcd07ffc4139a subpackages: @@ -20,12 +20,17 @@ imports: version: 941b50ebc6efddf4c41c8e4537a5f68a4e686b24 - name: github.com/mattn/go-isatty version: fc9e8d8ef48496124e79ae0df75490096eccf6fe +- name: github.com/SermoDigital/jose + version: b10f12c5188d355a3eacc61aba9abd3eb6b3b787 + subpackages: + - crypto + - jwt - name: github.com/valyala/bytebufferpool version: e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7 - name: github.com/valyala/fasttemplate version: dcecefd839c4193db0d35b88ec65b4c12d360ab0 - name: golang.org/x/crypto - version: 850760c427c516be930bc91280636328f1a62286 + version: adbae1b6b6fb4b02448a0fc0dbbc9ba2b95b294d subpackages: - acme - acme/autocert diff --git a/remote/models/ic/channel.go b/remote/models/ic/channel.go new file mode 100644 index 0000000000..abdd818770 --- /dev/null +++ b/remote/models/ic/channel.go @@ -0,0 +1,23 @@ +package ic + +// Channel is the representation of what a channel looks like in River Rock. +type Channel struct { + ID int + OrganizationID int +} + +func (c Channel) HostMaps(hosts []string, scope string) []*HostMap { + hostMaps := make([]*HostMap, len(hosts)) + + for idx, host := range hosts { + hostMap := &HostMap{ + Host: host, + ChannelID: c.ID, + Scope: scope, + } + + hostMaps[idx] = hostMap + } + + return hostMaps +} diff --git a/remote/models/ic/host_map.go b/remote/models/ic/host_map.go new file mode 100644 index 0000000000..67dd13872f --- /dev/null +++ b/remote/models/ic/host_map.go @@ -0,0 +1,10 @@ +package ic + +// HostMap stores the relationship been site hostnames and channels. +// Used by River Rock to determine which channel to use for a given request. +type HostMap struct { + ID int + Host string + ChannelID int + Scope string +} diff --git a/remote/models/phoenix/channel.go b/remote/models/phoenix/channel.go index ea90c0bd75..524b1803ce 100644 --- a/remote/models/phoenix/channel.go +++ b/remote/models/phoenix/channel.go @@ -9,16 +9,20 @@ import ( // Channel represents an avenue for purchasing on the Fox Platform. This could // be a website (theperfectgourmet.com), third-party (Amazon), or sale type (B2B). type Channel struct { - ID int - Name string - PurchaseLocation int - CreatedAt time.Time - UpdatedAt time.Time + ID int + IntelligenceChannelID int + Scope string + Name string + PurchaseLocation int + CreatedAt time.Time + UpdatedAt time.Time } -func NewChannel(name string, purchaseLocation int) *Channel { +func NewChannel(scope string, name string, purchaseLocation int) *Channel { return &Channel{ - ID: 0, + ID: 0, + IntelligenceChannelID: 0, + Scope: scope, Name: name, PurchaseLocation: purchaseLocation, CreatedAt: time.Now().UTC(), @@ -26,29 +30,6 @@ func NewChannel(name string, purchaseLocation int) *Channel { } } -func (c Channel) Table() string { - return "channels" -} - -func (c Channel) Fields() map[string]interface{} { - return map[string]interface{}{ - "name": c.Name, - "purchase_location": c.PurchaseLocation, - "created_at": c.CreatedAt, - "updated_at": c.UpdatedAt, - } -} - -func (c Channel) FieldRefs() []interface{} { - return []interface{}{ - &c.ID, - &c.Name, - &c.PurchaseLocation, - &c.CreatedAt, - &c.UpdatedAt, - } -} - func (c Channel) Validate() error { if c.Name == "" { return errors.New("Channel name must not be empty") diff --git a/remote/models/phoenix/organization.go b/remote/models/phoenix/organization.go new file mode 100644 index 0000000000..0b438998d2 --- /dev/null +++ b/remote/models/phoenix/organization.go @@ -0,0 +1,14 @@ +package phoenix + +import "time" + +type Organization struct { + ID int + Name string + Kind string + ParentID int + ScopeID int + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time +} diff --git a/remote/payloads/channel_payloads.go b/remote/payloads/channel_payloads.go index c0e372c698..de817a282b 100644 --- a/remote/payloads/channel_payloads.go +++ b/remote/payloads/channel_payloads.go @@ -3,15 +3,27 @@ package payloads import ( "time" + "github.com/FoxComm/highlander/remote/models/ic" "github.com/FoxComm/highlander/remote/models/phoenix" "github.com/FoxComm/highlander/remote/utils/failures" ) // CreateChannel is the structure of payload needed to create a channel. type CreateChannel struct { - Name string `json:"name"` - PurchaseOnFox *bool `json:"purchaseOnFox"` - CatalogID *int64 `json:"catalogId"` + Name string `json:"name"` + Scope string `json:"scope"` + OrganizationID int `json:"organizationId"` + Hosts []string `json:"hosts"` + PurchaseOnFox *bool `json:"purchaseOnFox"` + CatalogID *int64 `json:"catalogId"` +} + +// EnsureScope guarantees that a scope is set, either by using the scope that +// already exists on the payload, or setting a default scope from the caller. +func (c *CreateChannel) EnsureScope(scope string) { + if c.Scope == "" { + c.Scope = scope + } } // Validate ensures that the has the correct format. @@ -20,11 +32,22 @@ func (c CreateChannel) Validate() failures.Failure { return failures.NewFieldEmptyFailure("name") } else if c.PurchaseOnFox == nil { return failures.NewFieldEmptyFailure("purchaseOnFox") + } else if c.Scope == "" { + return failures.NewFieldEmptyFailure("scope") + } else if c.OrganizationID < 1 { + return failures.NewFieldGreaterThanZero("organizationId", c.OrganizationID) } return nil } +// IntelligenceModel returns the IC model for this payload. +func (c CreateChannel) IntelligenceModel() *ic.Channel { + return &ic.Channel{ + OrganizationID: c.OrganizationID, + } +} + // PhoenixModel returns the phoenix model for this payload. func (c CreateChannel) PhoenixModel() *phoenix.Channel { model := &phoenix.Channel{ @@ -41,3 +64,49 @@ func (c CreateChannel) PhoenixModel() *phoenix.Channel { return model } + +// UpdateChannel is the structure of payload needed to update a channel. +type UpdateChannel struct { + Name *string `json:"name"` + Hosts *[]string `json:"hosts"` + PurchaseOnFox *bool `json:"purchaseOnFox"` + CatalogID *int64 `json:"catalogId"` +} + +// Validate ensures that they payload has the correct format. +// For this payload, it's making sure that there's at least one value. +func (c UpdateChannel) Validate() failures.Failure { + if c.Name == nil && c.PurchaseOnFox == nil && c.CatalogID == nil { + return failures.NewEmptyPayloadFailure() + } else if c.Name != nil && (*c.Name) == "" { + return failures.NewFieldEmptyFailure("name") + } + + return nil +} + +func (c UpdateChannel) PhoenixModel(existing *phoenix.Channel) *phoenix.Channel { + newPhxChannel := phoenix.Channel{ + ID: existing.ID, + CreatedAt: existing.CreatedAt, + UpdatedAt: time.Now().UTC(), + } + + if c.Name != nil { + newPhxChannel.Name = *(c.Name) + } else { + newPhxChannel.Name = existing.Name + } + + if c.PurchaseOnFox != nil { + if *(c.PurchaseOnFox) { + newPhxChannel.PurchaseLocation = phoenix.PurchaseOnFox + } else { + newPhxChannel.PurchaseLocation = phoenix.PurchaseOffFox + } + } else { + newPhxChannel.PurchaseLocation = existing.PurchaseLocation + } + + return &newPhxChannel +} diff --git a/remote/payloads/scoped.go b/remote/payloads/scoped.go new file mode 100644 index 0000000000..38b431221c --- /dev/null +++ b/remote/payloads/scoped.go @@ -0,0 +1,5 @@ +package payloads + +type Scoped interface { + EnsureScope(string) +} diff --git a/remote/responses/channel.go b/remote/responses/channel.go index 5001200a27..9942c4b6c1 100644 --- a/remote/responses/channel.go +++ b/remote/responses/channel.go @@ -3,23 +3,33 @@ package responses import ( "time" + "github.com/FoxComm/highlander/remote/models/ic" "github.com/FoxComm/highlander/remote/models/phoenix" ) type Channel struct { - ID int `json:"id"` - Name string `json:"name"` - PurchaseOnFox bool `json:"purchaseOnFox"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID int `json:"id"` + OrganizationID int `json:"organizationId"` + Name string `json:"name"` + PurchaseOnFox bool `json:"purchaseOnFox"` + Hosts []string `json:"hosts"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } -func NewChannel(phxChannel *phoenix.Channel) *Channel { +func NewChannel(icChannel *ic.Channel, phxChannel *phoenix.Channel, hostMaps []*ic.HostMap) *Channel { + hosts := make([]string, len(hostMaps)) + for idx, hostMap := range hostMaps { + hosts[idx] = hostMap.Host + } + c := Channel{ - ID: phxChannel.ID, - Name: phxChannel.Name, - CreatedAt: phxChannel.CreatedAt, - UpdatedAt: phxChannel.UpdatedAt, + ID: phxChannel.ID, + OrganizationID: icChannel.OrganizationID, + Name: phxChannel.Name, + Hosts: hosts, + CreatedAt: phxChannel.CreatedAt, + UpdatedAt: phxChannel.UpdatedAt, } if phxChannel.PurchaseLocation == phoenix.PurchaseOnFox { diff --git a/remote/responses/health.go b/remote/responses/health.go new file mode 100644 index 0000000000..c2c3218b60 --- /dev/null +++ b/remote/responses/health.go @@ -0,0 +1,7 @@ +package responses + +// Health is the response that determines +type Health struct { + Intelligence string `json:"intelligence"` + Phoenix string `json:"phoenix"` +} diff --git a/remote/services/connection.go b/remote/services/connection.go index cb9dd0b4d3..6842b6266c 100644 --- a/remote/services/connection.go +++ b/remote/services/connection.go @@ -8,6 +8,23 @@ import ( _ "github.com/lib/pq" ) +func NewIntelligenceConnection(config *utils.Config) (*gorm.DB, error) { + connStr := fmt.Sprintf( + "host=%s user=%s password=%s dbname=%s sslmode=%s", + config.ICDatabaseHost, + config.ICDatabaseUser, + config.ICDatabasePassword, + config.ICDatabaseName, + config.ICDatabaseSSL) + + icDB, err := gorm.Open("postgres", connStr) + if err != nil { + return nil, fmt.Errorf("Unable to connect to IC DB with error %s", err.Error()) + } + + return icDB, nil +} + func NewPhoenixConnection(config *utils.Config) (*gorm.DB, error) { connStr := fmt.Sprintf( "host=%s user=%s password=%s dbname=%s sslmode=%s", diff --git a/remote/services/db.go b/remote/services/db.go new file mode 100644 index 0000000000..e689bf69be --- /dev/null +++ b/remote/services/db.go @@ -0,0 +1,125 @@ +package services + +import ( + "fmt" + "reflect" + + "github.com/FoxComm/highlander/remote/utils" + "github.com/FoxComm/highlander/remote/utils/failures" + "github.com/jinzhu/gorm" +) + +type RemoteDBs struct { + icDB *RemoteDB + phxDB *RemoteDB +} + +func NewRemoteDBs(config *utils.Config) (*RemoteDBs, error) { + icDB, err := NewIntelligenceConnection(config) + if err != nil { + return nil, err + } + + phxDB, err := NewPhoenixConnection(config) + if err != nil { + return nil, err + } + + return &RemoteDBs{ + icDB: &RemoteDB{icDB}, + phxDB: &RemoteDB{phxDB}, + }, nil +} + +func (r RemoteDBs) Begin() *RemoteDBs { + return &RemoteDBs{ + icDB: r.icDB.Begin(), + phxDB: r.phxDB.Begin(), + } +} + +func (r RemoteDBs) Commit() failures.Failure { + if fail := r.icDB.Commit(); fail != nil { + r.phxDB.Rollback() + return fail + } + + return r.phxDB.Commit() +} + +func (r RemoteDBs) Rollback() { + r.icDB.Rollback() + r.phxDB.Rollback() +} + +func (r RemoteDBs) IC() *RemoteDB { + return r.icDB +} + +func (r RemoteDBs) Phx() *RemoteDB { + return r.phxDB +} + +type RemoteDB struct { + db *gorm.DB +} + +func (r RemoteDB) Begin() *RemoteDB { + return &RemoteDB{db: r.db.Begin()} +} + +func (r RemoteDB) Commit() failures.Failure { + return failures.New(r.db.Commit().Error) +} + +func (r RemoteDB) Rollback() { + r.db.Rollback() +} + +func (r RemoteDB) Create(model interface{}) failures.Failure { + return failures.New(r.db.Create(model).Error) +} + +func (r RemoteDB) CreateWithTable(model interface{}, table string) failures.Failure { + return failures.New(r.db.Table(table).Create(model).Error) +} + +func (r RemoteDB) FindByID(id int, model interface{}) failures.Failure { + return r.FindByIDWithFailure(id, model, failures.FailureNotFound) +} + +func (r RemoteDB) FindByIDWithFailure(id int, model interface{}, failure int) failures.Failure { + res := r.db.First(model, id) + + if res.RecordNotFound() { + err := fmt.Errorf("%s with id %d was not found", typeName(model), id) + return failures.NewGeneralFailure(err, failure) + } + + return failures.New(res.Error) +} + +func (r RemoteDB) FindWhere(field string, value interface{}, models interface{}) failures.Failure { + res := r.db.Where(fmt.Sprintf("%s = ?", field), value).Find(models) + return failures.New(res.Error) +} + +func (r RemoteDB) Save(model interface{}) failures.Failure { + return failures.New(r.db.Save(model).Error) +} + +func (r RemoteDB) Delete(model interface{}) failures.Failure { + return failures.New(r.db.Delete(model).Error) +} + +func (r RemoteDB) Ping() error { + return r.db.DB().Ping() +} + +func typeName(i interface{}) string { + if t := reflect.TypeOf(i); t.Kind() == reflect.Ptr { + return t.Elem().Name() + } else { + return t.Name() + } +} diff --git a/remote/services/queries.go b/remote/services/queries.go index 4e9ee1bde4..919dad1a3a 100644 --- a/remote/services/queries.go +++ b/remote/services/queries.go @@ -1,20 +1,146 @@ package services import ( + "github.com/FoxComm/highlander/remote/models/ic" "github.com/FoxComm/highlander/remote/models/phoenix" + "github.com/FoxComm/highlander/remote/payloads" + "github.com/FoxComm/highlander/remote/responses" "github.com/FoxComm/highlander/remote/utils/failures" - "github.com/jinzhu/gorm" ) -func FindChannelByID(phxDB *gorm.DB, id int, phxChannel *phoenix.Channel) failures.Failure { - params := map[string]interface{}{ - "model": "channel", - "id": id, +func FindChannelByID(dbs *RemoteDBs, id int) (*responses.Channel, failures.Failure) { + var phxChannel *phoenix.Channel + if fail := dbs.Phx().FindByID(id, phxChannel); fail != nil { + return nil, fail } - return failures.New(phxDB.First(phxChannel, id).Error, params) + var icChannel *ic.Channel + fail := dbs.IC().FindByIDWithFailure( + phxChannel.IntelligenceChannelID, + icChannel, + failures.FailureBadRequest) + + if fail != nil { + return nil, fail + } + + var hostMaps []*ic.HostMap + if fail := dbs.IC().FindWhere("channel_id", icChannel.ID, &hostMaps); fail != nil { + return nil, fail + } + + return responses.NewChannel(icChannel, phxChannel, hostMaps), nil +} + +func InsertChannel(dbs *RemoteDBs, payload *payloads.CreateChannel) (*responses.Channel, failures.Failure) { + icChannel := payload.IntelligenceModel() + phxChannel := payload.PhoenixModel() + hostMaps := icChannel.HostMaps(payload.Hosts, phxChannel.Scope) + + var phxOrganization phoenix.Organization + + txn := dbs.Begin() + + if fail := txn.Phx().FindByIDWithFailure(icChannel.OrganizationID, &phxOrganization, failures.FailureBadRequest); fail != nil { + txn.Rollback() + return nil, fail + } + + if fail := txn.Phx().Create(phxChannel); fail != nil { + txn.Rollback() + return nil, fail + } + + if fail := txn.IC().Create(icChannel); fail != nil { + txn.Rollback() + return nil, fail + } + + // We have to iterate through each insert manually because of a limitation in + // Gorm. Since there will rarely be many hosts created at a time, this should + // be a workable solution for now. + for idx := range hostMaps { + if fail := txn.IC().CreateWithTable(hostMaps[idx], "host_map"); fail != nil { + txn.Rollback() + return nil, fail + } + } + + if fail := txn.Commit(); fail != nil { + return nil, fail + } + + return responses.NewChannel(icChannel, phxChannel, hostMaps), nil +} + +func compareHosts(currentHMs []*ic.HostMap, newHMs []string) ([]*ic.HostMap, []*ic.HostMap) { + existing := map[string]*ic.HostMap{} + for _, host := range currentHMs { + existing[host.Host] = host + } + + toAdd := []*ic.HostMap{} + for _, newHost := range newHMs { + if _, ok := existing[newHost]; !ok { + hm := &ic.HostMap{Host: newHost} + toAdd = append(toAdd, hm) + delete(existing, newHost) + } + } + + toRemove := []*ic.HostMap{} + for _, oldHost := range existing { + toRemove = append(toRemove, oldHost) + } + + return toAdd, toRemove } -func InsertChannel(phxDB *gorm.DB, phxChannel *phoenix.Channel) failures.Failure { - return failures.New(phxDB.Create(phxChannel).Error, nil) +func UpdateChannel(dbs *RemoteDBs, id int, payload *payloads.UpdateChannel) (*responses.Channel, failures.Failure) { + var origPhx *phoenix.Channel + if fail := dbs.Phx().FindByID(id, origPhx); fail != nil { + return nil, fail + } + + txn := dbs.Begin() + + newPhx := payload.PhoenixModel(origPhx) + if fail := txn.Phx().Save(newPhx); fail != nil { + txn.Rollback() + return fail + } + + if payload.Hosts != nil { + var origHost []*ic.HostMap + + } + + if fail := dbs.Commit(); fail != nil { + return nil, fail + } + + // if payload.Hosts != nil { + // newHost := *(payload.Hosts) + // toAdd, toDelete := compareHosts(origHost, newHost) + + // for _, host := range toAdd { + // host.ChannelID = origIC.ID + // host.Scope = newPhx.Scope + + // if fail := txn.IC().CreateWithTable(host, "host_map"); fail != nil { + // txn.Rollback() + // return fail + // } + // } + + // for _, host := range toDelete { + // if fail := txn.IC().Delete(host); fail != nil { + // txn.Rollback() + // return fail + // } + // } + // } + + // return dbs.Commit() + return nil, nil } diff --git a/remote/utils/config.go b/remote/utils/config.go index 5ab5627f98..f62595fa53 100644 --- a/remote/utils/config.go +++ b/remote/utils/config.go @@ -9,6 +9,12 @@ import ( const ( errEnvVarNotFound = "%s not found in the environment" + icDatabaseName = "IC_DATABASE_NAME" + icDatabaseHost = "IC_DATABASE_HOST" + icDatabaseUser = "IC_DATABASE_USER" + icDatabasePassword = "IC_DATABASE_PASSWORD" + icDatabaseSSL = "IC_DATABASE_SSL" + phoenixDatabaseName = "PHX_DATABASE_NAME" phoenixDatabaseHost = "PHX_DATABASE_HOST" phoenixDatabaseUser = "PHX_DATABASE_USER" @@ -19,6 +25,12 @@ const ( ) type Config struct { + ICDatabaseName string + ICDatabaseHost string + ICDatabaseUser string + ICDatabasePassword string + ICDatabaseSSL string + PhxDatabaseName string PhxDatabaseHost string PhxDatabaseUser string @@ -32,6 +44,12 @@ func NewConfig() (*Config, error) { config := &Config{} var err error + config.ICDatabaseName, err = parseEnvVar(icDatabaseName, nil) + config.ICDatabaseHost, err = parseEnvVar(icDatabaseHost, err) + config.ICDatabaseUser, err = parseEnvVar(icDatabaseUser, err) + config.ICDatabaseSSL, err = parseEnvVar(icDatabaseSSL, err) + config.ICDatabasePassword = os.Getenv(icDatabasePassword) + config.PhxDatabaseName, err = parseEnvVar(phoenixDatabaseName, nil) config.PhxDatabaseHost, err = parseEnvVar(phoenixDatabaseHost, err) config.PhxDatabaseUser, err = parseEnvVar(phoenixDatabaseUser, err) diff --git a/remote/utils/failures/failure.go b/remote/utils/failures/failure.go index f170a828b0..fd9fd78bd3 100644 --- a/remote/utils/failures/failure.go +++ b/remote/utils/failures/failure.go @@ -10,7 +10,7 @@ const ( FailureNotFound FailureServiceError - recordNotFound = "record not found" + RecordNotFound = "record not found" ) // Failure is a wrapper around the standard Golang error type that gives us more @@ -27,34 +27,24 @@ type Failure interface { } // New creates a new Failure. It determines the best failure based on the error -// and arguments passed in. If no error occurred, the response will be nil. -func New(err error, params map[string]interface{}) Failure { +// passed in. If no error occurred, the response will be nil. +func New(err error) Failure { if err == nil { return nil } stack := newCallStack() - if err.Error() == recordNotFound { - return newNotFoundFailure(stack, err, params) + if err.Error() == RecordNotFound { + return newNotFoundFailure(stack, err) } return newServiceFailure(stack, err) } -func newNotFoundFailure(stack *callStack, originalErr error, params map[string]interface{}) Failure { - model, modelOk := params["model"] - id, idOk := params["id"] - - var notFoundErr error - if !modelOk || !idOk { - notFoundErr = originalErr - } else { - notFoundErr = fmt.Errorf("%s with id %d was not found", model, id) - } - +func newNotFoundFailure(stack *callStack, originalErr error) Failure { return &generalFailure{ - err: notFoundErr, + err: originalErr, failureType: FailureNotFound, stack: stack, } diff --git a/remote/utils/failures/failures.go b/remote/utils/failures/failures.go index 6cd48c9667..7aa687f87a 100644 --- a/remote/utils/failures/failures.go +++ b/remote/utils/failures/failures.go @@ -1,6 +1,9 @@ package failures -import "fmt" +import ( + "errors" + "fmt" +) func NewParamNotFound(paramName string) Failure { return &generalFailure{ @@ -26,6 +29,14 @@ func NewBindFailure(err error) Failure { } } +func NewEmptyPayloadFailure() Failure { + return &generalFailure{ + err: errors.New("payload must have contents"), + failureType: FailureBadRequest, + stack: newCallStack(), + } +} + func NewFieldEmptyFailure(paramName string) Failure { return &generalFailure{ err: fmt.Errorf("%s must be non-empty", paramName), @@ -33,3 +44,19 @@ func NewFieldEmptyFailure(paramName string) Failure { stack: newCallStack(), } } + +func NewFieldGreaterThanZero(paramName string, value int) Failure { + return &generalFailure{ + err: fmt.Errorf("Expected %s to be greater than 0, got %d", paramName, value), + failureType: FailureBadRequest, + stack: newCallStack(), + } +} + +func NewModelNotFoundFailure(modelName string, id int) Failure { + return &generalFailure{ + err: fmt.Errorf("%s with id %d was not found", modelName, id), + failureType: FailureNotFound, + stack: newCallStack(), + } +} diff --git a/tabernacle/ansible/group_vars/all b/tabernacle/ansible/group_vars/all index 81b64626eb..428ec59c77 100644 --- a/tabernacle/ansible/group_vars/all +++ b/tabernacle/ansible/group_vars/all @@ -84,6 +84,7 @@ docker_tags: data_import: "{{ lookup('env', 'DOCKER_TAG_DATA_IMPORT') | default('master', true) }}" onboarding_service: "{{ lookup('env', 'DOCKER_TAG_ONBOARING_SERVICE') | default('master', true) }}" onboarding_ui: "{{ lookup('env', 'DOCKER_TAG_ONBOARING_UI') | default('master', true) }}" + remote: "{{ lookup('env', 'DOCKER_TAG_REMOTE') | default('master', true) }}" # Configurable Marathon re-deploys marathon_restart: @@ -119,6 +120,7 @@ marathon_restart: neo4j: "{{ lookup('env', 'MARATHON_NEO4J') | default(true, true) | bool }}" neo4j_reset: "{{ lookup('env', 'MARATHON_NEO4J_RESET') | default(true, true) | bool }}" suggester: "{{ lookup('env', 'MARATHON_SUGGESTER') | default(true, true) | bool }}" + remote: "{{ lookup('env', 'MARATHON_REMOTE') | default('master', true) | bool }}" # Should we do seeding for appliance detach_seeders: "{{ lookup('env', 'DETACH_SEEDERS') | default(false, true) | bool }}" @@ -224,6 +226,10 @@ hyperion_host: "hyperion.{{consul_suffix}}" hyperion_port: 8880 hyperion_server: "{{hyperion_host}}:{{hyperion_port}}" +remote_host: "remote.{{consul_suffix}}" +remote_port: 9898 +remote_server: "{{remote_host}}:{{remote_port}}" + # Database & bottledwater db_user: phoenix db_name: phoenix diff --git a/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 b/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 index 212210ec46..31e5a7eb96 100644 --- a/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 +++ b/tabernacle/ansible/roles/app/config_gen/templates/goldrush.cfg.j2 @@ -24,6 +24,7 @@ export DOCKER_TAG_MESSAGING:=master export DOCKER_TAG_ISAAC:=master export DOCKER_TAG_SOLOMON:=master export DOCKER_TAG_HYPERION:=master +export DOCKER_TAG_REMOTE:=master # Consumers export DOCKER_TAG_CAPTURE_CONSUMER:=master @@ -74,6 +75,7 @@ export MARATHON_MESSAGING:=false export MARATHON_ISAAC:=false export MARATHON_SOLOMON:=false export MARATHON_HYPERION:=false +export MARATHON_REMOTE:=false # Consumers export MARATHON_CAPTURE_CONSUMER:=false diff --git a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 index e3f60ed127..3a0d7e41fc 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/service_locations.j2 @@ -270,6 +270,16 @@ location /api/v1/hyperion/ { break; } +location /api/v1/tenant/ { + auth_request /internal-auth; + proxy_pass http://remote/v1/public/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + break; +} + location = /admin/styleguide { rewrite ^ https://{{storefront_server_name}}/admin/styleguide/ permanent; } diff --git a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 index c4a3f36e54..66d25c0e95 100644 --- a/tabernacle/ansible/roles/dev/balancer/templates/services.j2 +++ b/tabernacle/ansible/roles/dev/balancer/templates/services.j2 @@ -20,6 +20,7 @@ upstream eggcrate { << else >> server {{eggcrate_server}} fail_timeout=30s max_fails=10; << end >> } + upstream anthill { << range service "anthill" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; << else >> server {{anthill_server}} fail_timeout=30s max_fails=10; << end >> @@ -63,6 +64,11 @@ upstream hyperion { << else >> server {{hyperion_server}} fail_timeout=30s max_fails=10; << end >> } +upstream remote { + << range service "remote" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; + << else >> server {{remote_server}} fail_timeout=30s max_fails=10; << end >> +} + upstream ashes { << range service "ashes" >> server << .Address >>:<< .Port >> max_fails=10 fail_timeout=30s weight=1; << else >> server {{ashes_server}} fail_timeout=30s max_fails=10; << end >> diff --git a/tabernacle/ansible/roles/dev/marathon/tasks/app_remote.yml b/tabernacle/ansible/roles/dev/marathon/tasks/app_remote.yml new file mode 100644 index 0000000000..ae6f334699 --- /dev/null +++ b/tabernacle/ansible/roles/dev/marathon/tasks/app_remote.yml @@ -0,0 +1,25 @@ +--- + +- name: Copy Remote Marathon JSON + template: src=remote.json dest=/marathon/applications mode="u+rw,g+rw,o+r" + +- name: Kill Remote Tasks in Marathon + shell: 'curl -sS -XDELETE http://{{marathon_server}}/v2/apps/remote/tasks?scale=true' + when: is_redeploy + +- name: Pause for a bit... + pause: seconds=15 + when: is_redeploy + +- name: Update Remote in Marathon + shell: 'curl -sS -XPUT -d@/marathon/applications/remote.json -H "Content-Type: application/json" http://{{marathon_server}}/v2/apps/remote' + +- name: Restart Remote in Marathon + shell: 'curl -sS -XPOST http://{{marathon_server}}/v2/apps/remote/restart' + +- name: Get Remote Marathon tasks in `healthy` state + shell: curl -sS -XGET http://{{marathon_server}}/v2/apps/remote | jq '.app.tasksHealthy > 0' + register: healthy_tasks_available + until: healthy_tasks_available.stdout == 'true' + retries: "{{marathon_retries}}" + delay: "{{marathon_delay}}" diff --git a/tabernacle/ansible/roles/dev/marathon/tasks/backend.yml b/tabernacle/ansible/roles/dev/marathon/tasks/backend.yml index 38eced6b1c..79cdad2194 100644 --- a/tabernacle/ansible/roles/dev/marathon/tasks/backend.yml +++ b/tabernacle/ansible/roles/dev/marathon/tasks/backend.yml @@ -20,6 +20,10 @@ include: app_hyperion.yml when: marathon_restart.hyperion +- name: Start Remote + include: app_remote.yml + when: marathon_restart.remote + - name: Start Onboarding Service include: app_onboarding_service.yml when: with_onboarding and marathon_restart.onboarding_service diff --git a/tabernacle/ansible/roles/dev/marathon/templates/remote.json b/tabernacle/ansible/roles/dev/marathon/templates/remote.json new file mode 100644 index 0000000000..9aa5b4ec76 --- /dev/null +++ b/tabernacle/ansible/roles/dev/marathon/templates/remote.json @@ -0,0 +1,53 @@ +{ + "id": "/remote", + "cmd": null, + "cpus": 0.25, + "mem": 64, + "disk": 0, + "instances": 1, + "labels": { + "LAYER": "backend", + "LANG": "go", + "consul": "remote", + "TAG": "{{docker_tags.remote}}" + }, + "container": { + "type": "DOCKER", + "volumes": [ + { + "containerPath": "{{docker_logs_dir}}", + "hostPath": "{{docker_logs_host_dir}}", + "mode": "RW" + } + ], + "docker": { + "image": "{{docker_registry}}:5000/remote:{{docker_tags.remote}}", + "network": "HOST", + "privileged": false, + "parameters": [], + "forcePullImage": true + } + }, + "env": { + "IC_DATABASE_HOST": "{{docker_db_host}}", + "IC_DATABASE_NAME": "{{bernardo_db_name}}", + "IC_DATABASE_USER": "{{bernardo_db_user}}", + "IC_DATABASE_SSL": "disable", + "PHX_DATABASE_HOST": "{{docker_db_host}}", + "PHX_DATABASE_NAME": "{{phoenix_db_name}}", + "PHX_DATABASE_USER": "{{phoenix_db_user}}", + "PHX_DATABASE_SSL": "disable" + }, + "healthChecks": [ + { + "path": "/v1/public/health", + "protocol": "HTTP", + "gracePeriodSeconds": 300, + "intervalSeconds": 30, + "timeoutSeconds": 20, + "maxConsecutiveFailures": 3, + "ignoreHttp1xx": false, + "portIndex": 0 + } + ] +} diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/core-integrations/remote.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-integrations/remote.json.j2 new file mode 100644 index 0000000000..0d32ac22d9 --- /dev/null +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/core-integrations/remote.json.j2 @@ -0,0 +1,59 @@ +{ + "id": "remote", + "cmd": null, + "cpus": 0.25, + "mem": 64, + "disk": 0, + "instances": 1, + "constraints": [], + "labels": { + "MARATHON_SINGLE_INSTANCE_APP": "false", + "LANG": "go", + "consul": "remote", + "overrideTaskName": "remote", + "TAG": "{{docker_tags.remote}}" + }, + "upgradeStrategy": { + "minimumHealthCapacity": 0, + "maximumOverCapacity": 0 + }, + "container": { + "type": "DOCKER", + "volumes": [ + { + "containerPath": "{{docker_logs_dir}}", + "hostPath": "{{docker_logs_host_dir}}", + "mode": "RW" + } + ], + "docker": { + "image": "{{docker_registry}}:5000/remote:{{docker_tags.remote}}", + "network": "HOST", + "privileged": false, + "parameters": [], + "forcePullImage": true + } + }, + "env": { + "IC_DATABASE_HOST": "{{docker_db_host}}", + "IC_DATABASE_NAME": "{{bernardo_db_name}}", + "IC_DATABASE_USER": "{{bernardo_db_user}}", + "IC_DATABASE_SSL": "disable", + "PHX_DATABASE_HOST": "{{docker_db_host}}", + "PHX_DATABASE_NAME": "{{phoenix_db_name}}", + "PHX_DATABASE_USER": "{{phoenix_db_user}}", + "PHX_DATABASE_SSL": "disable" + }, + "healthChecks": [ + { + "path": "/v1/public/health", + "protocol": "HTTP", + "gracePeriodSeconds": 300, + "intervalSeconds": 30, + "timeoutSeconds": 20, + "maxConsecutiveFailures": 3, + "ignoreHttp1xx": false, + "portIndex": 0 + } + ] +} diff --git a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 index a38dbc77e8..6d60f25349 100644 --- a/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 +++ b/tabernacle/ansible/roles/dev/marathon_groups/templates/highlander.json.j2 @@ -50,6 +50,7 @@ {% include "core-integrations/shipstation.json.j2" %}, {% endif %} {% include "core-integrations/messaging.json.j2" %}, + {% include "core-integrations/remote.json.j2" %}, {% include "core-integrations/hyperion.json.j2" %} ] },