From 83d6ae7a4c4350b9a371418b054c90ba60715d79 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 12 Jul 2023 21:52:52 +0100 Subject: [PATCH 1/6] Initial --- go.mod | 3 ++ go.sum | 7 ++++ grpc/raw_receiver.proto | 34 ++++++++++++++++++ grpc_server.go | 80 +++++++++++++++++++++++++++++++++++++++++ main.go | 17 +++++++++ routes.go | 1 + update_grpc.sh | 4 +++ 7 files changed, 146 insertions(+) create mode 100644 grpc/raw_receiver.proto create mode 100644 grpc_server.go create mode 100755 update_grpc.sh diff --git a/go.mod b/go.mod index c98d3e13..1b4714c1 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/tidwall/rtree v1.10.0 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.30.0 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -42,6 +43,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.1 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -70,6 +72,7 @@ require ( golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.9.0 // indirect golang.org/x/text v0.10.0 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c5c3150b..5e49a90e 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I= github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -251,7 +253,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/grpc/raw_receiver.proto b/grpc/raw_receiver.proto new file mode 100644 index 00000000..fcd0d3a3 --- /dev/null +++ b/grpc/raw_receiver.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +option go_package = "github.com/unownhash/golbat/grpc"; + +package raw_receiver; + +// Interface exported by the server. +service RawProto { + rpc SubmitRawProto (RawProtoRequest) returns (RawProtoResponse) {} +} + +message RawProtoRequest { + string device_id = 1; + string username = 2; + + int32 trainer_level = 3; + float lat_target = 4; + float lon_target = 5; + int64 timestamp = 6; + optional bool have_ar = 7; + repeated Content contents = 8; + optional string scan_context = 9; +} + +message Content { + bytes response_payload = 1; + optional bytes request_payload = 2; + int32 method = 3; + optional bool have_ar = 4; +} + +message RawProtoResponse { + string message = 1; +} diff --git a/grpc_server.go b/grpc_server.go new file mode 100644 index 00000000..be0fcd27 --- /dev/null +++ b/grpc_server.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "golbat/config" + pb "golbat/grpc" + "time" +) + +// server is used to implement helloworld.GreeterServer. +type grpcServer struct { + pb.UnimplementedRawProtoServer +} + +func (s *grpcServer) SubmitRawProto(ctx context.Context, in *pb.RawProtoRequest) (*pb.RawProtoResponse, error) { + + //authHeader := r.Header.Get("Authorization") + //if config.Config.RawBearer != "" { + // if authHeader != "Bearer "+config.Config.RawBearer { + // log.Errorf("Raw: Incorrect authorisation received (%s)", authHeader) + // return + // } + //} + + uuid := in.DeviceId + account := in.Username + level := int(in.TrainerLevel) + scanContext := "" + if in.ScanContext != nil { + scanContext = *in.ScanContext + } + + latTarget, lonTarget := float64(in.LatTarget), float64(in.LonTarget) + globalHaveAr := in.HaveAr + var protoData []ProtoData + + for _, v := range in.Contents { + + inboundRawData := ProtoData{ + Method: int(v.Method), + Account: account, + Level: level, + ScanContext: scanContext, + Lat: latTarget, + Lon: lonTarget, + Data: v.ResponsePayload, + Request: v.RequestPayload, + Uuid: uuid, + HaveAr: func() *bool { + if v.HaveAr != nil { + return v.HaveAr + } + return globalHaveAr + }(), + } + + protoData = append(protoData, inboundRawData) + } + + // Process each proto in a packet in sequence, but in a go-routine + go func() { + timeout := 5 * time.Second + if config.Config.Tuning.ExtendedTimeout { + timeout = 30 * time.Second + } + + for _, entry := range protoData { + // provide independent cancellation contexts for each proto decode + ctx, cancel := context.WithTimeout(context.Background(), timeout) + decode(ctx, entry.Method, &entry) + cancel() + } + }() + + if latTarget != 0 && lonTarget != 0 && uuid != "" { + UpdateDeviceLocation(uuid, latTarget, lonTarget, scanContext) + } + + return &pb.RawProtoResponse{Message: "Processed"}, nil +} diff --git a/main.go b/main.go index 935015b2..a8c3674e 100644 --- a/main.go +++ b/main.go @@ -7,7 +7,10 @@ import ( db2 "golbat/db" "golbat/decoder" "golbat/external" + pb "golbat/grpc" "golbat/webhooks" + "google.golang.org/grpc" + "net" "time" _ "time/tzdata" @@ -172,6 +175,20 @@ func main() { go decoder.LoadAllGyms(dbDetails) } + // Start the GRPC receiver + + lis, err := net.Listen("tcp", ":50051") + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterRawProtoServer(s, &grpcServer{}) + log.Printf("grpc server listening at %v", lis.Addr()) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } + + // Start the web server. gin.SetMode(gin.ReleaseMode) r := gin.New() if config.Config.Logging.Debug { diff --git a/routes.go b/routes.go index ea9ff365..e5f1a45f 100644 --- a/routes.go +++ b/routes.go @@ -19,6 +19,7 @@ import ( ) type ProtoData struct { + Method int Data []byte Request []byte HaveAr *bool diff --git a/update_grpc.sh b/update_grpc.sh new file mode 100755 index 00000000..868d5c10 --- /dev/null +++ b/update_grpc.sh @@ -0,0 +1,4 @@ +protoc --go_out=. --go_opt=paths=source_relative \ + --go-grpc_out=. --go-grpc_opt=paths=source_relative \ + --go_opt=Mvbase.proto=github.com/unownhash/golbat/grpc \ + grpc/raw_receiver.proto \ No newline at end of file From 36874a8925099e712b19f7e1d083141c694f79bd Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 13 Jul 2023 10:04:17 +0100 Subject: [PATCH 2/6] Compression support --- grpc/raw_receiver.pb.go | 397 +++++++++++++++++++++++++++++++++++ grpc/raw_receiver_grpc.pb.go | 105 +++++++++ grpc_server.go | 1 + 3 files changed, 503 insertions(+) create mode 100644 grpc/raw_receiver.pb.go create mode 100644 grpc/raw_receiver_grpc.pb.go diff --git a/grpc/raw_receiver.pb.go b/grpc/raw_receiver.pb.go new file mode 100644 index 00000000..31a25d13 --- /dev/null +++ b/grpc/raw_receiver.pb.go @@ -0,0 +1,397 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.12 +// source: grpc/raw_receiver.proto + +package grpc + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type RawProtoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DeviceId string `protobuf:"bytes,1,opt,name=device_id,json=deviceId,proto3" json:"device_id,omitempty"` + Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"` + TrainerLevel int32 `protobuf:"varint,3,opt,name=trainer_level,json=trainerLevel,proto3" json:"trainer_level,omitempty"` + LatTarget float32 `protobuf:"fixed32,4,opt,name=lat_target,json=latTarget,proto3" json:"lat_target,omitempty"` + LonTarget float32 `protobuf:"fixed32,5,opt,name=lon_target,json=lonTarget,proto3" json:"lon_target,omitempty"` + Timestamp int64 `protobuf:"varint,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + HaveAr *bool `protobuf:"varint,7,opt,name=have_ar,json=haveAr,proto3,oneof" json:"have_ar,omitempty"` + Contents []*Content `protobuf:"bytes,8,rep,name=contents,proto3" json:"contents,omitempty"` + ScanContext *string `protobuf:"bytes,9,opt,name=scan_context,json=scanContext,proto3,oneof" json:"scan_context,omitempty"` +} + +func (x *RawProtoRequest) Reset() { + *x = RawProtoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_raw_receiver_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RawProtoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RawProtoRequest) ProtoMessage() {} + +func (x *RawProtoRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_raw_receiver_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RawProtoRequest.ProtoReflect.Descriptor instead. +func (*RawProtoRequest) Descriptor() ([]byte, []int) { + return file_grpc_raw_receiver_proto_rawDescGZIP(), []int{0} +} + +func (x *RawProtoRequest) GetDeviceId() string { + if x != nil { + return x.DeviceId + } + return "" +} + +func (x *RawProtoRequest) GetUsername() string { + if x != nil { + return x.Username + } + return "" +} + +func (x *RawProtoRequest) GetTrainerLevel() int32 { + if x != nil { + return x.TrainerLevel + } + return 0 +} + +func (x *RawProtoRequest) GetLatTarget() float32 { + if x != nil { + return x.LatTarget + } + return 0 +} + +func (x *RawProtoRequest) GetLonTarget() float32 { + if x != nil { + return x.LonTarget + } + return 0 +} + +func (x *RawProtoRequest) GetTimestamp() int64 { + if x != nil { + return x.Timestamp + } + return 0 +} + +func (x *RawProtoRequest) GetHaveAr() bool { + if x != nil && x.HaveAr != nil { + return *x.HaveAr + } + return false +} + +func (x *RawProtoRequest) GetContents() []*Content { + if x != nil { + return x.Contents + } + return nil +} + +func (x *RawProtoRequest) GetScanContext() string { + if x != nil && x.ScanContext != nil { + return *x.ScanContext + } + return "" +} + +type Content struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ResponsePayload []byte `protobuf:"bytes,1,opt,name=response_payload,json=responsePayload,proto3" json:"response_payload,omitempty"` + RequestPayload []byte `protobuf:"bytes,2,opt,name=request_payload,json=requestPayload,proto3,oneof" json:"request_payload,omitempty"` + Method int32 `protobuf:"varint,3,opt,name=method,proto3" json:"method,omitempty"` + HaveAr *bool `protobuf:"varint,4,opt,name=have_ar,json=haveAr,proto3,oneof" json:"have_ar,omitempty"` +} + +func (x *Content) Reset() { + *x = Content{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_raw_receiver_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Content) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Content) ProtoMessage() {} + +func (x *Content) ProtoReflect() protoreflect.Message { + mi := &file_grpc_raw_receiver_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Content.ProtoReflect.Descriptor instead. +func (*Content) Descriptor() ([]byte, []int) { + return file_grpc_raw_receiver_proto_rawDescGZIP(), []int{1} +} + +func (x *Content) GetResponsePayload() []byte { + if x != nil { + return x.ResponsePayload + } + return nil +} + +func (x *Content) GetRequestPayload() []byte { + if x != nil { + return x.RequestPayload + } + return nil +} + +func (x *Content) GetMethod() int32 { + if x != nil { + return x.Method + } + return 0 +} + +func (x *Content) GetHaveAr() bool { + if x != nil && x.HaveAr != nil { + return *x.HaveAr + } + return false +} + +type RawProtoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` +} + +func (x *RawProtoResponse) Reset() { + *x = RawProtoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_raw_receiver_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RawProtoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RawProtoResponse) ProtoMessage() {} + +func (x *RawProtoResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_raw_receiver_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RawProtoResponse.ProtoReflect.Descriptor instead. +func (*RawProtoResponse) Descriptor() ([]byte, []int) { + return file_grpc_raw_receiver_proto_rawDescGZIP(), []int{2} +} + +func (x *RawProtoResponse) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +var File_grpc_raw_receiver_proto protoreflect.FileDescriptor + +var file_grpc_raw_receiver_proto_rawDesc = []byte{ + 0x0a, 0x17, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x61, 0x77, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, + 0x76, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x72, 0x61, 0x77, 0x5f, 0x72, + 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x22, 0xe1, 0x02, 0x0a, 0x0f, 0x52, 0x61, 0x77, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x64, + 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, + 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x74, 0x72, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, + 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, 0x52, 0x09, 0x6c, + 0x61, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x6e, 0x5f, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, 0x52, 0x09, 0x6c, 0x6f, + 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x07, 0x68, 0x61, 0x76, 0x65, 0x5f, 0x61, 0x72, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x06, 0x68, 0x61, 0x76, 0x65, 0x41, 0x72, + 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x61, 0x77, 0x5f, 0x72, 0x65, 0x63, 0x65, + 0x69, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0c, 0x73, 0x63, 0x61, 0x6e, 0x5f, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x0b, + 0x73, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x88, 0x01, 0x01, 0x42, 0x0a, + 0x0a, 0x08, 0x5f, 0x68, 0x61, 0x76, 0x65, 0x5f, 0x61, 0x72, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x73, + 0x63, 0x61, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xb8, 0x01, 0x0a, 0x07, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x12, 0x2c, 0x0a, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0e, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x88, 0x01, 0x01, + 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x1c, 0x0a, 0x07, 0x68, 0x61, 0x76, 0x65, + 0x5f, 0x61, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x48, 0x01, 0x52, 0x06, 0x68, 0x61, 0x76, + 0x65, 0x41, 0x72, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x68, + 0x61, 0x76, 0x65, 0x5f, 0x61, 0x72, 0x22, 0x2c, 0x0a, 0x10, 0x52, 0x61, 0x77, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x32, 0x5d, 0x0a, 0x08, 0x52, 0x61, 0x77, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x52, 0x61, 0x77, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x1d, 0x2e, 0x72, 0x61, 0x77, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, + 0x72, 0x2e, 0x52, 0x61, 0x77, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1e, 0x2e, 0x72, 0x61, 0x77, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, + 0x2e, 0x52, 0x61, 0x77, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x75, 0x6e, 0x6f, 0x77, 0x6e, 0x68, 0x61, 0x73, 0x68, 0x2f, 0x67, 0x6f, 0x6c, 0x62, + 0x61, 0x74, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_raw_receiver_proto_rawDescOnce sync.Once + file_grpc_raw_receiver_proto_rawDescData = file_grpc_raw_receiver_proto_rawDesc +) + +func file_grpc_raw_receiver_proto_rawDescGZIP() []byte { + file_grpc_raw_receiver_proto_rawDescOnce.Do(func() { + file_grpc_raw_receiver_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_raw_receiver_proto_rawDescData) + }) + return file_grpc_raw_receiver_proto_rawDescData +} + +var file_grpc_raw_receiver_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_grpc_raw_receiver_proto_goTypes = []interface{}{ + (*RawProtoRequest)(nil), // 0: raw_receiver.RawProtoRequest + (*Content)(nil), // 1: raw_receiver.Content + (*RawProtoResponse)(nil), // 2: raw_receiver.RawProtoResponse +} +var file_grpc_raw_receiver_proto_depIdxs = []int32{ + 1, // 0: raw_receiver.RawProtoRequest.contents:type_name -> raw_receiver.Content + 0, // 1: raw_receiver.RawProto.SubmitRawProto:input_type -> raw_receiver.RawProtoRequest + 2, // 2: raw_receiver.RawProto.SubmitRawProto:output_type -> raw_receiver.RawProtoResponse + 2, // [2:3] is the sub-list for method output_type + 1, // [1:2] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_grpc_raw_receiver_proto_init() } +func file_grpc_raw_receiver_proto_init() { + if File_grpc_raw_receiver_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_raw_receiver_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RawProtoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_raw_receiver_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Content); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_raw_receiver_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RawProtoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_raw_receiver_proto_msgTypes[0].OneofWrappers = []interface{}{} + file_grpc_raw_receiver_proto_msgTypes[1].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_raw_receiver_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_raw_receiver_proto_goTypes, + DependencyIndexes: file_grpc_raw_receiver_proto_depIdxs, + MessageInfos: file_grpc_raw_receiver_proto_msgTypes, + }.Build() + File_grpc_raw_receiver_proto = out.File + file_grpc_raw_receiver_proto_rawDesc = nil + file_grpc_raw_receiver_proto_goTypes = nil + file_grpc_raw_receiver_proto_depIdxs = nil +} diff --git a/grpc/raw_receiver_grpc.pb.go b/grpc/raw_receiver_grpc.pb.go new file mode 100644 index 00000000..287e3a31 --- /dev/null +++ b/grpc/raw_receiver_grpc.pb.go @@ -0,0 +1,105 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.21.12 +// source: grpc/raw_receiver.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// RawProtoClient is the client API for RawProto service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type RawProtoClient interface { + SubmitRawProto(ctx context.Context, in *RawProtoRequest, opts ...grpc.CallOption) (*RawProtoResponse, error) +} + +type rawProtoClient struct { + cc grpc.ClientConnInterface +} + +func NewRawProtoClient(cc grpc.ClientConnInterface) RawProtoClient { + return &rawProtoClient{cc} +} + +func (c *rawProtoClient) SubmitRawProto(ctx context.Context, in *RawProtoRequest, opts ...grpc.CallOption) (*RawProtoResponse, error) { + out := new(RawProtoResponse) + err := c.cc.Invoke(ctx, "/raw_receiver.RawProto/SubmitRawProto", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// RawProtoServer is the server API for RawProto service. +// All implementations must embed UnimplementedRawProtoServer +// for forward compatibility +type RawProtoServer interface { + SubmitRawProto(context.Context, *RawProtoRequest) (*RawProtoResponse, error) + mustEmbedUnimplementedRawProtoServer() +} + +// UnimplementedRawProtoServer must be embedded to have forward compatible implementations. +type UnimplementedRawProtoServer struct { +} + +func (UnimplementedRawProtoServer) SubmitRawProto(context.Context, *RawProtoRequest) (*RawProtoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubmitRawProto not implemented") +} +func (UnimplementedRawProtoServer) mustEmbedUnimplementedRawProtoServer() {} + +// UnsafeRawProtoServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RawProtoServer will +// result in compilation errors. +type UnsafeRawProtoServer interface { + mustEmbedUnimplementedRawProtoServer() +} + +func RegisterRawProtoServer(s grpc.ServiceRegistrar, srv RawProtoServer) { + s.RegisterService(&RawProto_ServiceDesc, srv) +} + +func _RawProto_SubmitRawProto_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RawProtoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RawProtoServer).SubmitRawProto(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/raw_receiver.RawProto/SubmitRawProto", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RawProtoServer).SubmitRawProto(ctx, req.(*RawProtoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// RawProto_ServiceDesc is the grpc.ServiceDesc for RawProto service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var RawProto_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "raw_receiver.RawProto", + HandlerType: (*RawProtoServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SubmitRawProto", + Handler: _RawProto_SubmitRawProto_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/raw_receiver.proto", +} diff --git a/grpc_server.go b/grpc_server.go index be0fcd27..58f4ee2b 100644 --- a/grpc_server.go +++ b/grpc_server.go @@ -4,6 +4,7 @@ import ( "context" "golbat/config" pb "golbat/grpc" + _ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor "time" ) From 5b286e55e183c8d5073e81ec482bab3f69fd410d Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 19 Jul 2023 20:17:40 +0100 Subject: [PATCH 3/6] Specify grpc port in config --- config.toml.example | 1 + config/config.go | 1 + main.go | 21 ++++++++++++--------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/config.toml.example b/config.toml.example index 09d98289..880e7c6c 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,4 +1,5 @@ port = 9001 # Listening port for golbat +#grpc_port = 50001 # Listening port for grpc raw_bearer = "" # Raw bearer (password) required api_secret = "golbat" # Golbat secret required on api calls (blank for none) diff --git a/config/config.go b/config/config.go index c7638ff3..6184b73b 100644 --- a/config/config.go +++ b/config/config.go @@ -4,6 +4,7 @@ import "golbat/geo" type configDefinition struct { Port int `koanf:"port"` + GrpcPort int `koanf:"grpc_port"` Webhooks []webhook `koanf:"webhooks"` Database database `koanf:"database"` Stats bool `koanf:"stats"` diff --git a/main.go b/main.go index a8c3674e..b8e99962 100644 --- a/main.go +++ b/main.go @@ -177,15 +177,18 @@ func main() { // Start the GRPC receiver - lis, err := net.Listen("tcp", ":50051") - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - s := grpc.NewServer() - pb.RegisterRawProtoServer(s, &grpcServer{}) - log.Printf("grpc server listening at %v", lis.Addr()) - if err := s.Serve(lis); err != nil { - log.Fatalf("failed to serve: %v", err) + if config.Config.GrpcPort > 0 { + log.Infof("Starting GRPC server on port %d", config.Config.GrpcPort) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Config.GrpcPort)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterRawProtoServer(s, &grpcServer{}) + log.Printf("grpc server listening at %v", lis.Addr()) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } } // Start the web server. From 989a2015445aa6ef67096479ca4a3e35583e4038 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 19 Jul 2023 21:04:16 +0100 Subject: [PATCH 4/6] Start in goroutine --- main.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index b8e99962..a75bc2d7 100644 --- a/main.go +++ b/main.go @@ -179,16 +179,18 @@ func main() { if config.Config.GrpcPort > 0 { log.Infof("Starting GRPC server on port %d", config.Config.GrpcPort) - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Config.GrpcPort)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - s := grpc.NewServer() - pb.RegisterRawProtoServer(s, &grpcServer{}) - log.Printf("grpc server listening at %v", lis.Addr()) - if err := s.Serve(lis); err != nil { - log.Fatalf("failed to serve: %v", err) - } + go func() { + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Config.GrpcPort)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterRawProtoServer(s, &grpcServer{}) + log.Printf("grpc server listening at %v", lis.Addr()) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } + }() } // Start the web server. From ef9b78f7db0c387251f8e1859f4a25c6c017ad09 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 20 Jul 2023 14:56:10 +0100 Subject: [PATCH 5/6] Remove local cache when did not write to db --- decoder/pokemon.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 2dd6f708..2ebeca2a 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -310,6 +310,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po if err != nil { log.Errorf("insert pokemon: [%s] %s", pokemon.Id, err) log.Errorf("Full structure: %+v", pokemon) + pokemonCache.Delete(pokemon.Id) // Force reload of pokemon from database return } @@ -360,12 +361,12 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po if err != nil { log.Errorf("Update pokemon [%s] %s", pokemon.Id, err) log.Errorf("Full structure: %+v", pokemon) + pokemonCache.Delete(pokemon.Id) // Force reload of pokemon from database + return } rows, rowsErr := res.RowsAffected() log.Debugf("Updating pokemon [%s] after update res = %d %v", pokemon.Id, rows, rowsErr) - - _, _ = res, err } } From d29257e551c7b252e9b6cf11905725caa77d3b63 Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 24 Jul 2023 16:07:28 +0100 Subject: [PATCH 6/6] Check password for raw bearer --- grpc_server.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/grpc_server.go b/grpc_server.go index 58f4ee2b..703bd0a1 100644 --- a/grpc_server.go +++ b/grpc_server.go @@ -5,6 +5,7 @@ import ( "golbat/config" pb "golbat/grpc" _ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor + "google.golang.org/grpc/metadata" "time" ) @@ -14,14 +15,14 @@ type grpcServer struct { } func (s *grpcServer) SubmitRawProto(ctx context.Context, in *pb.RawProtoRequest) (*pb.RawProtoResponse, error) { + // Check for authorisation + if config.Config.RawBearer != "" { + md, _ := metadata.FromIncomingContext(ctx) - //authHeader := r.Header.Get("Authorization") - //if config.Config.RawBearer != "" { - // if authHeader != "Bearer "+config.Config.RawBearer { - // log.Errorf("Raw: Incorrect authorisation received (%s)", authHeader) - // return - // } - //} + if auth := md.Get("authorization"); len(auth) == 0 || auth[0] != config.Config.RawBearer { + return &pb.RawProtoResponse{Message: "Incorrect authorisation received"}, nil + } + } uuid := in.DeviceId account := in.Username