Create the scaffolding for a simple webserver to fulfill dialogflow r…

…equests. (#143)

* This is intended to be a simple web server that will answer queries
  about who owns which GitHub labels.

* This PR is just the scaffolding for the server

  * The server provides an endpoint for the Dialogflow webhook but
    the endpoint isn't actually returning valid responses yet.

Related to #142

.gitignore

@@ -14,6 +14,7 @@ fairing/__pycache__/**
 *.pyc
 py/code_intelligence/.data/**
 
+**/.idea
 # ignore coredumps
 **/core.*
 # ignore checkpoints

chatbot/Dockerfile

@@ -0,0 +1,27 @@
+# Build the manager binary
+ARG GOLANG_VERSION=1.13.1
+FROM golang:${GOLANG_VERSION} as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY . ./
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o server ./cmd/main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/base:latest as serve
+WORKDIR /
+COPY --from=builder /workspace/server /server
+
+EXPOSE 8080
+
+ENTRYPOINT ["/server"]

chatbot/README.md

@@ -0,0 +1,75 @@
+# Chatbot
+
+This is a Dialogflow fulfillment server.
+
+## Deployment
+
+It is currently running in
+
+* **Full cluster name**: gke_issue-label-bot-dev_us-east1-d_issue-label-bot
+* **project**: issue-label-bot-dev
+* **cluster**: issue-label-bot
+
+## Notes.
+
+To expose the webhook we need to bypass IAP. To do this we create a second K8s service to create a second GCP Backend Service
+but with IAP enabled.
+
+```
+kubectl --context=issue-label-bot-dev  -n istio-system create -f istio-ingressgateway.yaml
+```
+
+We need to modify the security policy applied at the ingress gateway so that it won't reject requests without a valid
+JWT.
+
+To deploy the fullfilment server we need to modify the Kubeflow ingress  policy to allow traffic from the dialgoflow webserver.
+This traffic can't be routed through IAP. We will still use a JWT to restrict traffic but it will be a JWT we create.
+
+So we need to add a second JWT origin rule to match this traffic to the policy.
+
+We can do this as
+
+```
+kubectl --context=issue-label-bot-dev -n istio-system patch policy  ingress-jwt -p "$(cat ingress-jwt.patch.yaml)" --type=merge
+```
+
+To verify that is working we can port-forward to the service.
+
+```
+kubectl --context=issue-label-bot-dev  -n istio-system port-forward service/chatbot-istio-ingressgateway 9080:80
+```
+
+Send a request with a JWT this should fail with "Origin Authentication Failure" since there is no JWT.
+
+```
+curl localhost:9080/chatbot/dev/  -d '{}' -H "Content-Type: application/json" 
+```
+
+
+
+To authorize Dialogflow webhook we will use a JWT.  We use the jose-util to generate a public private key pair
+
+```
+git clone [email protected]:square/go-jose.git git_go-jose
+cd git_go-jose/jose-uitl
+go build.
+```
+
+Generate a key pair
+
+
+```
+./jose-util generate-key --alg=ES256 --use sig --kid=chatbot
+```
+
+Upload the public bit to a public GCS bucket
+
+```
+https://storage.cloud.google.com/issue-label-bot-dev_public/chatbot/keys/jwk-sig-chatbot-pub.json
+```
+
+## Referencess
+
+* [ISTIO 1.1 Policy Resource](https://archive.istio.io/v1.1/docs/reference/config/istio.authentication.v1alpha1/#Policy)
+* [ISTIO 1.5 JWT policy example](https://istio.io/docs/tasks/security/authorization/authz-jwt/)
+  * This example includes some static JWTs that can be used for testing.

chatbot/cmd/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"flag"
+	"github.com/kubeflow/code-intelligence/chatbot/cmd/options"
+	"github.com/kubeflow/code-intelligence/chatbot/pkg"
+	"github.com/onrik/logrus/filename"
+	log "github.com/sirupsen/logrus"
+)
+
+func init() {
+	// Add filename as one of the fields of the structured log message
+	filenameHook := filename.NewHook()
+	filenameHook.Field = "filename"
+	log.AddHook(filenameHook)
+}
+
+// Run the application.
+func Run(opt *options.ServerOption) error {
+	log.Info("Creating server")
+	server, err := pkg.NewKubeflowInfoServer(opt.AreaConfigPath)
+	if err != nil {
+		return err
+	}
+
+
+	server.RegisterEndpoints()
+
+	log.Infof("Starting http server.")
+	return server.StartHttp(opt.Port)
+}
+
+func main() {
+	s := options.NewServerOption()
+	s.AddFlags(flag.CommandLine)
+
+	flag.Parse()
+
+	if s.AreaConfigPath == "" {
+		log.Fatalf("--area-config-path must be specified. This should be the path to a YAML file defining the areas and their associated owners")
+	}
+	if s.JsonLogFormat {
+		// Output logs in a json format so that it can be parsed by services like Stackdriver
+		log.SetFormatter(&log.JSONFormatter{})
+	}
+	if err := Run(s); err != nil {
+		log.Fatalf("%v\n", err)
+	}
+}

chatbot/cmd/options/options.go

@@ -0,0 +1,41 @@
+
+// Copyright 2018 The Kubeflow Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	"flag"
+)
+
+// ServerOption is the main context object for terver.
+type ServerOption struct {
+	PrintVersion         bool
+	JsonLogFormat        bool
+	AreaConfigPath       string
+	Port                 int
+}
+
+// NewServerOption creates a new CMServer with a default config.
+func NewServerOption() *ServerOption {
+	s := ServerOption{}
+	return &s
+}
+
+// AddFlags adds flags for a specific Server to the specified FlagSet
+func (s *ServerOption) AddFlags(fs *flag.FlagSet) {
+	fs.BoolVar(&s.JsonLogFormat, "json-log-format", true, "Set true to use json style log format. Set false to use plaintext style log format")
+	fs.StringVar(&s.AreaConfigPath, "area-config-path", "https://raw.githubusercontent.com/kubeflow/community/master/labels-owners.yaml", "Path to the YAML file mapping area labels to owners.")
+	fs.IntVar(&s.Port, "port", 8080, "The port to use for an http server.")
+}

chatbot/go.mod

@@ -0,0 +1,17 @@
+module github.com/kubeflow/code-intelligence/chatbot
+
+go 1.13
+
+require (
+	github.com/ghodss/yaml v1.0.0
+	github.com/go-kit/kit v0.9.0
+	github.com/hashicorp/go-getter v1.4.1
+	github.com/onrik/logrus v0.5.1
+	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.6.0
+	github.com/sirupsen/logrus v1.6.0
+	github.com/square/go-jose/v3 v3.0.0-20200430180204-d84c719419c2
+	github.com/tidwall/gjson v1.6.0 // indirect
+	golang.org/x/net v0.0.0-20190620200207-3b0461eec859
+	gopkg.in/yaml.v2 v2.2.8 // indirect
+)