ensure decoded json data is valid utf-8 (#94)

codec/abi_decoder.go

@@ -25,6 +25,7 @@ import (
 	"github.com/streamingfast/bstream"
 	"go.uber.org/zap"
 	"math"
+	"strings"
 )
 
 var mostRecentActiveABI uint64 = math.MaxUint64
@@ -504,7 +505,7 @@ func (d *ABIDecoder) decodeActionTrace(actionTrace *pbantelope.ActionTrace, glob
 			return nil
 		}
 
-		actionTrace.JsonReturnValue = string(res)
+		actionTrace.JsonReturnValue = strings.ToValidUTF8(string(res), "�")
 	}
 
 	return nil
@@ -597,8 +598,8 @@ func (d *ABIDecoder) decodeAction(action *pbantelope.Action, globalSequence uint
 		return nil
 	}
 
+	action.JsonData = strings.ToValidUTF8(string(jsonData), "�")
 	zlog.Debug("successfully decoded action data", zap.String("action", action.SimpleName()), zap.String("json_data", string(jsonData)))
-	action.JsonData = string(jsonData)
 
 	return nil
 }
@@ -657,7 +658,7 @@ func (d *ABIDecoder) decodeDbOp(dbOp *pbantelope.DBOp, globalSequence uint64, tr
 			return nil
 		}
 
-		dbOp.OldDataJson = string(oldDataJson)
+		dbOp.OldDataJson = strings.ToValidUTF8(string(oldDataJson), "�")
 		zlog.Debug("decoded old data", zap.String("old_json_data", dbOp.OldDataJson))
 	}
 
@@ -689,7 +690,7 @@ func (d *ABIDecoder) decodeDbOp(dbOp *pbantelope.DBOp, globalSequence uint64, tr
 			return nil
 		}
 
-		dbOp.NewDataJson = string(newDataJson)
+		dbOp.NewDataJson = strings.ToValidUTF8(string(newDataJson), "�")
 		zlog.Debug("decoded new data", zap.String("new_json_data", dbOp.NewDataJson))
 	}