From 94516a65a1a6d22644d6f60aff87fb3c129bf6db Mon Sep 17 00:00:00 2001 From: Sandro Mello Date: Thu, 17 Nov 2022 10:18:12 -0300 Subject: [PATCH] Add DLP info type configuration (#56) * Add DLP info type configuration --- agent/dlp/gcp.go | 33 +++++++++++++++------- agent/dlp/types.go | 37 ++----------------------- agent/exec.go | 7 ++++- common/proto/types.go | 7 +++-- gateway/api/server.go | 4 ++- gateway/transport/client.go | 28 +++++++++++++------ gateway/transport/plugins/dlp/dlp.go | 41 ++++++++++++++++++++++++++++ 7 files changed, 98 insertions(+), 59 deletions(-) create mode 100644 gateway/transport/plugins/dlp/dlp.go diff --git a/agent/dlp/gcp.go b/agent/dlp/gcp.go index 8bbd4782..701927f9 100644 --- a/agent/dlp/gcp.go +++ b/agent/dlp/gcp.go @@ -33,12 +33,20 @@ func NewDLPClient(ctx context.Context, credentialsJSON []byte) (*Client, error) func NewDLPStreamWriter(client pb.ClientTransport, dlpClient *Client, packetType pb.PacketType, - spec map[string][]byte) *streamWriter { + spec map[string][]byte, + infoTypeList []string) *streamWriter { + dlpConfig := &DeidentifyConfig{ + MaskingCharacter: defaultMaskingCharacter, + NumberToMask: defaultNumberToMask, + InfoTypes: parseInfoTypes(infoTypeList), + ProjectID: dlpClient.GetProjectID(), + } return &streamWriter{ client: client, dlpClient: dlpClient, packetType: packetType, packetSpec: spec, + dlpConfig: dlpConfig, } } @@ -49,9 +57,9 @@ func (s *streamWriter) Write(data []byte) (int, error) { } p.Type = s.packetType.String() p.Spec = s.packetSpec - if s.dlpClient != nil && len(data) > 30 { + if s.dlpClient != nil && len(data) > 30 && len(s.dlpConfig.InfoTypes) > 0 { chunksBuffer := breakPayloadIntoChunks(bytes.NewBuffer(data)) - redactedChunks := redactChunks(s.dlpClient, s.dlpClient.GetProjectID(), chunksBuffer) + redactedChunks := redactChunks(s.dlpClient, s.dlpConfig, chunksBuffer) dataBuffer, tsList, err := joinChunks(redactedChunks) if err != nil { return 0, fmt.Errorf("failed joining chunks, err=%v", err) @@ -138,13 +146,7 @@ func deidentifyContent(ctx context.Context, client *Client, conf *DeidentifyConf // redactChunks process chunks in parallel reordering after the end of each execution. // A default timeout is applied for each chunk. If a requests timeout or returns an error the chunk is returned // without redacting its content. -func redactChunks(client *Client, projectID string, chunksBuffer []*bytes.Buffer) []*Chunk { - conf := &DeidentifyConfig{ - MaskingCharacter: defaultMaskingCharacter, - NumberToMask: defaultNumberToMask, - InfoTypes: defaultInfoTypes, - ProjectID: projectID, - } +func redactChunks(client *Client, conf *DeidentifyConfig, chunksBuffer []*bytes.Buffer) []*Chunk { chunkCh := make(chan *Chunk) for idx, chunkBuf := range chunksBuffer { go func(idx int, chunkB *bytes.Buffer) { @@ -200,3 +202,14 @@ func breakPayloadIntoChunks(payload *bytes.Buffer) []*bytes.Buffer { } return chunks } + +func parseInfoTypes(infoTypesList []string) []*dlppb.InfoType { + var infoTypes []*dlppb.InfoType + for _, infoType := range infoTypesList { + if infoType == "" { + continue + } + infoTypes = append(infoTypes, &dlppb.InfoType{Name: infoType}) + } + return infoTypes +} diff --git a/agent/dlp/types.go b/agent/dlp/types.go index e6127edb..88b6c35b 100644 --- a/agent/dlp/types.go +++ b/agent/dlp/types.go @@ -18,41 +18,6 @@ const ( maxChunkSize = 62500 ) -var defaultInfoTypes = []*dlppb.InfoType{ - {Name: "PHONE_NUMBER"}, - {Name: "AGE"}, - {Name: "CREDIT_CARD_NUMBER"}, - {Name: "CREDIT_CARD_TRACK_NUMBER"}, - {Name: "DATE_OF_BIRTH"}, - {Name: "EMAIL_ADDRESS"}, - {Name: "ETHNIC_GROUP"}, - {Name: "GENDER"}, - {Name: "IBAN_CODE"}, - {Name: "HTTP_COOKIE"}, - {Name: "ICD9_CODE"}, - {Name: "ICD10_CODE"}, - {Name: "IMEI_HARDWARE_ID"}, - {Name: "IP_ADDRESS"}, - {Name: "STORAGE_SIGNED_URL"}, - {Name: "URL"}, - {Name: "VEHICLE_IDENTIFICATION_NUMBER"}, - {Name: "BRAZIL_CPF_NUMBER"}, - {Name: "AMERICAN_BANKERS_CUSIP_ID"}, - {Name: "FDA_CODE"}, - {Name: "US_ADOPTION_TAXPAYER_IDENTIFICATION_NUMBER"}, - {Name: "US_BANK_ROUTING_MICR"}, - {Name: "US_DEA_NUMBER"}, - {Name: "US_DRIVERS_LICENSE_NUMBER"}, - {Name: "US_EMPLOYER_IDENTIFICATION_NUMBER"}, - {Name: "US_HEALTHCARE_NPI"}, - {Name: "US_INDIVIDUAL_TAXPAYER_IDENTIFICATION_NUMBER"}, - {Name: "US_PASSPORT"}, - {Name: "US_PREPARER_TAXPAYER_IDENTIFICATION_NUMBER"}, - {Name: "US_SOCIAL_SECURITY_NUMBER"}, - {Name: "US_TOLLFREE_PHONE_NUMBER"}, - {Name: "US_VEHICLE_IDENTIFICATION_NUMBER"}, -} - type ( TransformationSummary struct { Index int @@ -76,6 +41,8 @@ type ( dlpClient *Client packetType pb.PacketType packetSpec map[string][]byte + infoTypes []string + dlpConfig *DeidentifyConfig } DeidentifyConfig struct { // Character to use to mask the sensitive values, for example, `*` for an diff --git a/agent/exec.go b/agent/exec.go index aa2bd22c..dfff2a41 100644 --- a/agent/exec.go +++ b/agent/exec.go @@ -95,7 +95,12 @@ func (a *Agent) doExecWriteAgentStdin(pkt *pb.Packet) { } stdoutWriter := pb.NewStreamWriter(a.client, pb.PacketExecClientWriteStdoutType, spec) if dlpClient, ok := a.connStore.Get(dlpClientKey).(*dlp.Client); ok { - stdoutWriter = dlp.NewDLPStreamWriter(a.client, dlpClient, pb.PacketExecClientWriteStdoutType, spec) + stdoutWriter = dlp.NewDLPStreamWriter( + a.client, + dlpClient, + pb.PacketExecClientWriteStdoutType, + spec, + connParams.DLPInfoTypes) } if err := cmd.RunOnTTY(stdoutWriter, onExecEnd); err != nil { log.Printf("session=%s, tty=true - err=%v", string(sessionID), err) diff --git a/common/proto/types.go b/common/proto/types.go index 5b62c707..6c5b3119 100644 --- a/common/proto/types.go +++ b/common/proto/types.go @@ -30,9 +30,10 @@ type ( packetSpec map[string][]byte } AgentConnectionParams struct { - EnvVars map[string]any - CmdList []string - ClientArgs []string + EnvVars map[string]any + CmdList []string + ClientArgs []string + DLPInfoTypes []string } ) diff --git a/gateway/api/server.go b/gateway/api/server.go index 2861adca..d8e27218 100644 --- a/gateway/api/server.go +++ b/gateway/api/server.go @@ -2,10 +2,11 @@ package api import ( "fmt" - "github.com/runopsio/hoop/gateway/review" "os" "strings" + "github.com/runopsio/hoop/gateway/review" + "github.com/runopsio/hoop/gateway/security" "github.com/runopsio/hoop/gateway/security/idp" @@ -108,6 +109,7 @@ func (api *Api) CreateTrialEntities() error { Name: "hooper", Email: "tester@hoop.dev", Status: "active", + Groups: []string{"admin"}, } a := agent.Agent{ diff --git a/gateway/transport/client.go b/gateway/transport/client.go index 9f3306ae..6bdd2eda 100644 --- a/gateway/transport/client.go +++ b/gateway/transport/client.go @@ -2,10 +2,6 @@ package transport import ( "fmt" - "github.com/runopsio/hoop/gateway/plugin" - pluginsaudit "github.com/runopsio/hoop/gateway/transport/plugins/audit" - pluginsreview "github.com/runopsio/hoop/gateway/transport/plugins/review" - "github.com/runopsio/hoop/gateway/user" "io" "log" "os" @@ -13,6 +9,12 @@ import ( "sync" "syscall" + "github.com/runopsio/hoop/gateway/plugin" + pluginsaudit "github.com/runopsio/hoop/gateway/transport/plugins/audit" + pluginsdlp "github.com/runopsio/hoop/gateway/transport/plugins/dlp" + pluginsreview "github.com/runopsio/hoop/gateway/transport/plugins/review" + "github.com/runopsio/hoop/gateway/user" + "github.com/google/uuid" pb "github.com/runopsio/hoop/common/proto" "github.com/runopsio/hoop/gateway/client" @@ -57,6 +59,7 @@ func LoadPlugins() { allPlugins = []Plugin{ pluginsaudit.New(), pluginsreview.New(), + pluginsdlp.New(), } } @@ -247,10 +250,18 @@ func (s *Server) processPacketGatewayConnect(pkt *pb.Packet, } } } + var infoTypes []string + for _, p := range getPlugins(client.SessionID) { + if p.Plugin.Name() == pluginsdlp.Name { + infoTypes = p.config + break + } + } encConnectionParams, err := pb.GobEncode(&pb.AgentConnectionParams{ - EnvVars: conn.Secret, - CmdList: conn.Command, - ClientArgs: clientArgs, + EnvVars: conn.Secret, + CmdList: conn.Command, + ClientArgs: clientArgs, + DLPInfoTypes: infoTypes, }) if err != nil { return fmt.Errorf("failed encoding connection params err=%v", err) @@ -295,8 +306,7 @@ func (s *Server) clientGracefulShutdown(c *client.Client) { syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, - syscall.SIGQUIT, - syscall.SIGKILL) + syscall.SIGQUIT) go func() { <-sigc s.disconnectClient(c) diff --git a/gateway/transport/plugins/dlp/dlp.go b/gateway/transport/plugins/dlp/dlp.go new file mode 100644 index 00000000..5a063b12 --- /dev/null +++ b/gateway/transport/plugins/dlp/dlp.go @@ -0,0 +1,41 @@ +package dlp + +import ( + "github.com/runopsio/hoop/gateway/plugin" + + pb "github.com/runopsio/hoop/common/proto" +) + +const Name string = "dlp" + +type ( + dlpPlugin struct { + name string + } +) + +func New() *dlpPlugin { + return &dlpPlugin{name: Name} +} + +func (p *dlpPlugin) Name() string { + return p.name +} + +func (p *dlpPlugin) OnStartup(config plugin.Config) error { + return nil +} + +func (p *dlpPlugin) OnConnect(config plugin.Config) error { + return nil +} + +func (p *dlpPlugin) OnReceive(pluginConfig plugin.Config, config []string, pkt *pb.Packet) error { + return nil +} + +func (p *dlpPlugin) OnDisconnect(config plugin.Config) error { + return nil +} + +func (p *dlpPlugin) OnShutdown() {}