From 8992be9fc5056cf254fe53efee1fb01c4019595f Mon Sep 17 00:00:00 2001 From: Charles Banning Date: Wed, 14 Mar 2018 15:29:25 -0600 Subject: [PATCH] add ExistingJSONKeys() --- README | 1 + existingkeys.go | 203 +++++++++++++++++++++++++++++++++++++++++++ existingkeys_test.go | 193 ++++++++++++++++++++++++++++++++++++++++ missingkeys.go | 25 +++--- 4 files changed, 409 insertions(+), 13 deletions(-) create mode 100644 existingkeys.go create mode 100644 existingkeys_test.go diff --git a/README b/README index 87b187b..068c792 100644 --- a/README +++ b/README @@ -2,6 +2,7 @@ Check that a JSON object's keys correspond to a struct's exported members or JSO ANNOUNCEMENTS +2018.03.14 - Add ExistingJSONKeys() 2018.02.16 - Add test example of using go v1.10 (*Decoder)DisallowUnknownFields() 2017.02.13 - Handle "-" and "omitempty" JSON tags in struct definitions. 2017.02.08 - UnknownJSONKeys lists all JSON object keys that won't be decoded. diff --git a/existingkeys.go b/existingkeys.go new file mode 100644 index 0000000..c768045 --- /dev/null +++ b/existingkeys.go @@ -0,0 +1,203 @@ +// existingkeys.go - check JSON object against struct definition +// Copyright © 2016-2018 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package checkjson + +import ( + "encoding/json" + "reflect" + "strings" +) + +// ExistingJSONKeys returns a list of fields of the struct 'val' that WILL BE set +// by unmarshaling the JSON object. It is the complement of MissingJSONKeys. +// For nested structs, field labels are the dot-notation hierachical +// path for a JSON key. Specific struct fields can be igored +// when scanning the JSON object by declaring them using SetMembersToIgnore. +// (NOTE: JSON object keys are treated as case insensitive, i.e., there +// is no distiction between "key":"value" and "Key":"value".) +// +// For embedded structs, both the field label for the embedded struct as well +// as the dot-notation label for that struct's fields are included in the list. Thus, +// type Person struct { +// Name NameInfo +// Sex string +// } +// +// type NameInfo struct { +// First, Middle, Last string +// } +// +// jobj := []byte(`{"name":{"first":"Jonnie","middle":"Q","last":"Public"},"sex":"unkown"}`) +// p := Person{} +// +// fields, _ := ExistingKeys(jobj, p) +// fmt.Println(fields) // prints: [Name Name.First Name.Middle Name.Last Sex] +// +// Struct fields that have JSON tag "-" are never returned. Struct fields with the tag +// attribute "omitempty" will, by default NOT be returned unless the keys exist in the JSON object. +// If you want to know if "omitempty" struct fields are actually in the JSON object, then call +// IgnoreOmitEmptyTag(false) prior to using ExistingJSONKeys. +func ExistingJSONKeys(b []byte, val interface{}) ([]string, error) { + s := make([]string, 0) + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return s, ResolveJSONError(b, err) + } + findMembers(m, reflect.ValueOf(val), &s, "") + return s, nil +} + +// cmem is the parent struct member for nested structs +func findMembers(mv interface{}, val reflect.Value, s *[]string, cmem string) { + // 1. Convert any pointer value. + if val.Kind() == reflect.Ptr { + val = reflect.Indirect(val) // convert ptr to struc + } + // zero Value? + if !val.IsValid() { + return + } + typ := val.Type() + + // json.RawMessage is a []byte/[]uint8 and has Kind() == reflect.Slice + if typ.Name() == "RawMessage" { + return + } + + // 2. If its a slice then 'mv' should hold a []interface{} value. + // Loop through the members of 'mv' and see that they are valid relative + // to the of val []. + if typ.Kind() == reflect.Slice { + tval := typ.Elem() + if tval.Kind() == reflect.Ptr { + tval = tval.Elem() + } + // slice may be nil, so create a Value of it's type + // 'mv' must be of type []interface{} + sval := reflect.New(tval) + slice, ok := mv.([]interface{}) + if !ok { + // encoding/json must have a JSON array value to decode + // unlike encoding/xml which will decode a list of elements + // to a singleton or vise-versa. + *s = append(*s, typ.Name()) + return + } + // 2.1. Check members of JSON array. + // This forces all of them to be regular and w/o typos in key labels. + for _, sl := range slice { + // cmem is the member name for the slice - [] - value + findMembers(sl, sval, s, cmem) + } + return // done with reflect.Slice value + } + + // 3a. Ignore anything that's not a struct. + if typ.Kind() != reflect.Struct { + return // just ignore it - don't look for k:v pairs + } + // 3b. map value must represent k:v pairs + mm, ok := mv.(map[string]interface{}) + if !ok { + *s = append(*s, typ.Name()) + } + // 3c. Coerce keys to lower case. + mkeys := make(map[string]interface{}, len(mm)) + for k, v := range mm { + mkeys[strings.ToLower(k)] = v + } + + // 4. Build the list of struct field name:value + // We make every key (field) label look like an exported label - "Fieldname". + // If there is a JSON tag it is used instead of the field label, and saved to + // insure that the spec'd tag matches the JSON key exactly. + type fieldSpec struct { + name string + val reflect.Value + tag string + omitempty bool + } + fields := make([]*fieldSpec, 0) // use a list so members are in sequence + var tag string + var oempty bool + for i := 0; i < val.NumField(); i++ { + tag = "" + oempty = false + if len(typ.Field(i).PkgPath) > 0 { + continue // field is NOT exported + } + t := typ.Field(i).Tag.Get("json") + tags := strings.Split(t, ",") + tag = tags[0] + // handle ignore member JSON tag, "-" + if tag == "-" { + continue + } + // scan rest of tags for "omitempty" + for _, v := range tags[1:] { + if v == "omitempty" { + oempty = true + break + } + } + if tag == "" { + fields = append(fields, &fieldSpec{typ.Field(i).Name, val.Field(i), "", oempty}) + } else { + fields = append(fields, &fieldSpec{typ.Field(i).Name, val.Field(i), tag, oempty}) + } + } + + // 5. check that field names/tags have corresponding map key + // var ok bool + var v interface{} + // var err error + cmemdepth := 1 + if len(cmem) > 0 { + cmemdepth = len(strings.Split(cmem, ".")) + 1 // struct hierarchy + } + lcmem := strings.ToLower(cmem) + name := "" + for _, field := range fields { + lm := strings.ToLower(field.name) + for _, sm := range skipmembers { + // skip any skipmembers values that aren't at same depth + if cmemdepth != sm.depth { + continue + } + if len(cmem) > 0 { + if lcmem+`.`+lm == sm.val { + goto next + } + } else if lm == sm.val { + goto next + } + } + if len(field.tag) > 0 { + name = field.tag + v, ok = mkeys[field.tag] + } else { + name = field.name + v, ok = mkeys[lm] + } + // If map key is missing, then record it + // if there's no omitempty tag or we're ignoring omitempty tag. + if !ok && (!field.omitempty || !omitemptyOK) { + goto next // don't drill down further; no key in JSON object + } + // field exists in JSON object, so add to list + if len(cmem) > 0 { + *s = append(*s, cmem+`.`+name) + } else { + *s = append(*s, name) + } + if len(cmem) > 0 { + findMembers(v, field.val, s, cmem+`.`+name) + } else { + findMembers(v, field.val, s, name) + } + next: + } +} diff --git a/existingkeys_test.go b/existingkeys_test.go new file mode 100644 index 0000000..17c18d1 --- /dev/null +++ b/existingkeys_test.go @@ -0,0 +1,193 @@ +package checkjson + +import ( + "fmt" + "testing" +) + +func TestExistingJSONKeys(t *testing.T) { + fmt.Println("===================== TestExistingJSONKeys ...") + + type test struct { + Ok bool + Why string + } + tv := test{} + data := []byte(`{"ok":true,"why":"it's a test"}`) + mems, err := ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + // returns: [Ok Why] + if len(mems) != 2 { + t.Fatalf(fmt.Sprintf("len(mems) == %d >> %v", len(mems), mems)) + } + + data = []byte(`{"ok":true}`) + mems, err = ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 1 { + t.Fatalf(fmt.Sprintf("existing mems: %d - %#v", len(mems), mems)) + } + if mems[0] != "Ok" { + t.Fatalf(fmt.Sprintf("existing keys: %v", mems)) + } +} +func TestExistingJSONSubKeys(t *testing.T) { + fmt.Println("===================== TestExistingJSONSubKeys ...") + type test3 struct { + Something string + Else string + } + + type test2 struct { + Why string + Not string + Another test3 + } + + type test struct { + Ok bool + Why string + More test2 + } + tv := test{} + data := []byte(`{"ok":true,"why":"it's a test","more":{"why":"again","not":"no more","another":{"something":"a thing","else":"ok"}}}`) + mems, err := ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 8 { + t.Fatalf(fmt.Sprintf("len(mems) == %d >> %v", len(mems), mems)) + } + + data = []byte(`{"ok":true,"more":{"why":"again","another":{"else":"ok"}}}`) + mems, err = ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 5 { + t.Fatalf(fmt.Sprintf("existing mems: %d - %#v", len(mems), mems)) + } + fmt.Println("existing keys:", mems) +} + +func TestExistingJSONKeysWithTags(t *testing.T) { + fmt.Println("===================== TestExistingJSONKeysWithTags ...") + + type test struct { + Ok bool + Why string `json:"whynot"` + } + tv := test{} + data := []byte(`{"ok":true,"whynot":"it's a test"}`) + mems, err := ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 2 { + t.Fatalf(fmt.Sprintf("len(mems) == %d >> %v", len(mems), mems)) + } + fmt.Println("existing keys:", mems) + + data = []byte(`{"ok":true,"why":"it's not a test"}`) + mems, err = ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 1 || mems[0] != "Ok" { + t.Fatalf(fmt.Sprintf("existing mems: %d - %#v", len(mems), mems)) + } + fmt.Println("existing keys:", mems) +} + +func TestExistingJSONKeysSkipMems(t *testing.T) { + fmt.Println("===================== TestExistingJSONKeysSkipMems ...") + + type test3 struct { + Something string + Else string + } + + type test2 struct { + Why string + Not string + Another test3 + } + + type test struct { + Ok bool + Why string + More test2 + } + + tv := test{} + data := []byte(`{"ok":true,"more":{"why":"again","another":{"else":"ok"}}}`) + SetMembersToIgnore("ok", "more.why") + defer SetMembersToIgnore() + + mems, err := ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 3 { + t.Fatalf(fmt.Sprintf("existing mems: %d - %#v", len(mems), mems)) + } + fmt.Println("existing keys:", mems) +} + +func TestExistingJSONKeysWithIgnoreTag(t *testing.T) { + fmt.Println("===================== TestExistingJSONKeysWithIgnoreTag ...") + + type test struct { + Ok bool + Why string + Whynot string `json:"-"` + } + tv := test{} + data := []byte(`{"ok":true,"why":"it's a test"}`) + mems, err := ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 2 { + t.Fatalf(fmt.Sprintf("len(mems) == %d >> %v", len(mems), mems)) + } + fmt.Println("existing keys:", mems) +} + +func TestExistingJSONKeysWithOmitemptyAttr(t *testing.T) { + fmt.Println("===================== TestExistingJSONKeysWithmitemptyIgnoreAttr ...") + + type test struct { + Ok bool + Why string + Whynot string `json:",omitempty"` + } + tv := test{} + data := []byte(`{"ok":true,"why":"it's a test"}`) + + IgnoreOmitemptyTag(true) // make sure it's set + mems, err := ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 3 { + t.Fatalf(fmt.Sprintf("len(mems) == %d >> %v", len(mems), mems)) + } + if mems[2] != "Whynot" { + t.Fatalf("ExistingJSONKeys did't get: Whynot") + } + + IgnoreOmitemptyTag(false) // ignore attribute + defer IgnoreOmitemptyTag(true) // reset on return + mems, err = ExistingJSONKeys(data, tv) + if err != nil { + t.Fatalf(err.Error()) + } + if len(mems) != 2 { + t.Fatalf(fmt.Sprintf("len(mems) == %d >> %v", len(mems), mems)) + } +} diff --git a/missingkeys.go b/missingkeys.go index ecea7c3..2037a53 100644 --- a/missingkeys.go +++ b/missingkeys.go @@ -16,10 +16,10 @@ type skipmems struct { depth int } -// Slice of dot-notation struct members that can be missing in JSON object. +// Slice of dot-notation struct fields that can be missing in JSON object. var skipmembers = []skipmems{} -// SetMembersToIgnore creates a list of exported struct member names that should not be checked +// SetMembersToIgnore creates a list of exported struct field names that should not be checked // for as keys in the JSON object. For hierarchical struct members provide the full path for // the member name using dot-notation. Calling SetMembersToIgnore with no arguments - // SetMembersToIgnore() - clears the list. @@ -39,9 +39,9 @@ var omitemptyOK = true // IgnoreOmitemptyTag determines whether a `json:",omitempty"` tag is recognized or // not with respect to the JSON object. By default MissingJSONKeys will not include -// struct members that are tagged with "omitempty" in the list of missing JSON keys. +// struct fields that are tagged with "omitempty" in the list of missing JSON keys. // If the function is toggled or passed the optional argument 'false' then missing -// JSON keys may include those for struct members with the 'omitempty' JSON tag. +// JSON keys may include those for struct fields with the 'omitempty' JSON tag. // // Calling IgnoreOmitemptyTag with no arguments toggles the handling on/off. If // the alternative argument is passed, then the argument value determines the @@ -54,19 +54,20 @@ func IgnoreOmitemptyTag(ok ...bool) { omitemptyOK = ok[0] } -// MissingJSONKeys returns a list of members of val of struct type that will NOT be set +// MissingJSONKeys returns a list of fields of a struct that will NOT be set // by unmarshaling the JSON object; rather, they will assume their default -// values. For nested structs, member labels are the dot-notation hierachical -// path for the missing JSON key. Specific struct members can be igored +// values. For nested structs, field labels are the dot-notation hierachical +// path for the missing JSON key. Specific struct fields can be igored // when scanning the JSON object by declaring them using SetMembersToIgnore(). // (NOTE: JSON object keys are treated as case insensitive, i.e., there // is no distiction between "key":"value" and "Key":"value".) // -// By default keys in the JSON object that are associated with struct members that -// have JSON tags "-" and "omitempty" are not included in the returned slice. +// By default keys in the JSON object that are associated with struct fields that +// have JSON tag "-" are ignored. If the "omitempty" attribute is included in the +// struct field tag they are by default also not included in the returned slice. // IgnoreOmitemptyTag(false) can be called to override the handling of "omitempty" -// tags - this might be useful if you want to find the "omitempty" members that -// are not set by decoding the JSON object.. +// tags - this might be useful if you want to find the "omitempty" fields that +// are not set by decoding the JSON object. func MissingJSONKeys(b []byte, val interface{}) ([]string, error) { s := make([]string, 0) m := make(map[string]interface{}) @@ -222,10 +223,8 @@ func checkMembers(mv interface{}, val reflect.Value, s *[]string, cmem string) { goto next // don't drill down further; no key in JSON object } if len(cmem) > 0 { - // checkMembers(v, field.val, s, cmem+`.`+field.name) checkMembers(v, field.val, s, cmem+`.`+name) } else { - // checkMembers(v, field.val, s, field.name) checkMembers(v, field.val, s, name) } next: