hashstructure.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. package hashstructure
  2. import (
  3. "encoding/binary"
  4. "fmt"
  5. "hash"
  6. "hash/fnv"
  7. "reflect"
  8. "time"
  9. )
  10. // HashOptions are options that are available for hashing.
  11. type HashOptions struct {
  12. // Hasher is the hash function to use. If this isn't set, it will
  13. // default to FNV.
  14. Hasher hash.Hash64
  15. // TagName is the struct tag to look at when hashing the structure.
  16. // By default this is "hash".
  17. TagName string
  18. // ZeroNil is flag determining if nil pointer should be treated equal
  19. // to a zero value of pointed type. By default this is false.
  20. ZeroNil bool
  21. // IgnoreZeroValue is determining if zero value fields should be
  22. // ignored for hash calculation.
  23. IgnoreZeroValue bool
  24. // SlicesAsSets assumes that a `set` tag is always present for slices.
  25. // Default is false (in which case the tag is used instead)
  26. SlicesAsSets bool
  27. // UseStringer will attempt to use fmt.Stringer always. If the struct
  28. // doesn't implement fmt.Stringer, it'll fall back to trying usual tricks.
  29. // If this is true, and the "string" tag is also set, the tag takes
  30. // precedence (meaning that if the type doesn't implement fmt.Stringer, we
  31. // panic)
  32. UseStringer bool
  33. }
  34. // Format specifies the hashing process used. Different formats typically
  35. // generate different hashes for the same value and have different properties.
  36. type Format uint
  37. const (
  38. // To disallow the zero value
  39. formatInvalid Format = iota
  40. // FormatV1 is the format used in v1.x of this library. This has the
  41. // downsides noted in issue #18 but allows simultaneous v1/v2 usage.
  42. FormatV1
  43. // FormatV2 is the current recommended format and fixes the issues
  44. // noted in FormatV1.
  45. FormatV2
  46. formatMax // so we can easily find the end
  47. )
  48. // Hash returns the hash value of an arbitrary value.
  49. //
  50. // If opts is nil, then default options will be used. See HashOptions
  51. // for the default values. The same *HashOptions value cannot be used
  52. // concurrently. None of the values within a *HashOptions struct are
  53. // safe to read/write while hashing is being done.
  54. //
  55. // The "format" is required and must be one of the format values defined
  56. // by this library. You should probably just use "FormatV2". This allows
  57. // generated hashes uses alternate logic to maintain compatibility with
  58. // older versions.
  59. //
  60. // Notes on the value:
  61. //
  62. // * Unexported fields on structs are ignored and do not affect the
  63. // hash value.
  64. //
  65. // * Adding an exported field to a struct with the zero value will change
  66. // the hash value.
  67. //
  68. // For structs, the hashing can be controlled using tags. For example:
  69. //
  70. // struct {
  71. // Name string
  72. // UUID string `hash:"ignore"`
  73. // }
  74. //
  75. // The available tag values are:
  76. //
  77. // * "ignore" or "-" - The field will be ignored and not affect the hash code.
  78. //
  79. // * "set" - The field will be treated as a set, where ordering doesn't
  80. // affect the hash code. This only works for slices.
  81. //
  82. // * "string" - The field will be hashed as a string, only works when the
  83. // field implements fmt.Stringer
  84. //
  85. func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
  86. // Validate our format
  87. if format <= formatInvalid || format >= formatMax {
  88. return 0, &ErrFormat{}
  89. }
  90. // Create default options
  91. if opts == nil {
  92. opts = &HashOptions{}
  93. }
  94. if opts.Hasher == nil {
  95. opts.Hasher = fnv.New64()
  96. }
  97. if opts.TagName == "" {
  98. opts.TagName = "hash"
  99. }
  100. // Reset the hash
  101. opts.Hasher.Reset()
  102. // Create our walker and walk the structure
  103. w := &walker{
  104. format: format,
  105. h: opts.Hasher,
  106. tag: opts.TagName,
  107. zeronil: opts.ZeroNil,
  108. ignorezerovalue: opts.IgnoreZeroValue,
  109. sets: opts.SlicesAsSets,
  110. stringer: opts.UseStringer,
  111. }
  112. return w.visit(reflect.ValueOf(v), nil)
  113. }
  114. type walker struct {
  115. format Format
  116. h hash.Hash64
  117. tag string
  118. zeronil bool
  119. ignorezerovalue bool
  120. sets bool
  121. stringer bool
  122. }
  123. type visitOpts struct {
  124. // Flags are a bitmask of flags to affect behavior of this visit
  125. Flags visitFlag
  126. // Information about the struct containing this field
  127. Struct interface{}
  128. StructField string
  129. }
  130. var timeType = reflect.TypeOf(time.Time{})
  131. func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
  132. t := reflect.TypeOf(0)
  133. // Loop since these can be wrapped in multiple layers of pointers
  134. // and interfaces.
  135. for {
  136. // If we have an interface, dereference it. We have to do this up
  137. // here because it might be a nil in there and the check below must
  138. // catch that.
  139. if v.Kind() == reflect.Interface {
  140. v = v.Elem()
  141. continue
  142. }
  143. if v.Kind() == reflect.Ptr {
  144. if w.zeronil {
  145. t = v.Type().Elem()
  146. }
  147. v = reflect.Indirect(v)
  148. continue
  149. }
  150. break
  151. }
  152. // If it is nil, treat it like a zero.
  153. if !v.IsValid() {
  154. v = reflect.Zero(t)
  155. }
  156. // Binary writing can use raw ints, we have to convert to
  157. // a sized-int, we'll choose the largest...
  158. switch v.Kind() {
  159. case reflect.Int:
  160. v = reflect.ValueOf(int64(v.Int()))
  161. case reflect.Uint:
  162. v = reflect.ValueOf(uint64(v.Uint()))
  163. case reflect.Bool:
  164. var tmp int8
  165. if v.Bool() {
  166. tmp = 1
  167. }
  168. v = reflect.ValueOf(tmp)
  169. }
  170. k := v.Kind()
  171. // We can shortcut numeric values by directly binary writing them
  172. if k >= reflect.Int && k <= reflect.Complex64 {
  173. // A direct hash calculation
  174. w.h.Reset()
  175. err := binary.Write(w.h, binary.LittleEndian, v.Interface())
  176. return w.h.Sum64(), err
  177. }
  178. switch v.Type() {
  179. case timeType:
  180. w.h.Reset()
  181. b, err := v.Interface().(time.Time).MarshalBinary()
  182. if err != nil {
  183. return 0, err
  184. }
  185. err = binary.Write(w.h, binary.LittleEndian, b)
  186. return w.h.Sum64(), err
  187. }
  188. switch k {
  189. case reflect.Array:
  190. var h uint64
  191. l := v.Len()
  192. for i := 0; i < l; i++ {
  193. current, err := w.visit(v.Index(i), nil)
  194. if err != nil {
  195. return 0, err
  196. }
  197. h = hashUpdateOrdered(w.h, h, current)
  198. }
  199. return h, nil
  200. case reflect.Map:
  201. var includeMap IncludableMap
  202. if opts != nil && opts.Struct != nil {
  203. if v, ok := opts.Struct.(IncludableMap); ok {
  204. includeMap = v
  205. }
  206. }
  207. // Build the hash for the map. We do this by XOR-ing all the key
  208. // and value hashes. This makes it deterministic despite ordering.
  209. var h uint64
  210. for _, k := range v.MapKeys() {
  211. v := v.MapIndex(k)
  212. if includeMap != nil {
  213. incl, err := includeMap.HashIncludeMap(
  214. opts.StructField, k.Interface(), v.Interface())
  215. if err != nil {
  216. return 0, err
  217. }
  218. if !incl {
  219. continue
  220. }
  221. }
  222. kh, err := w.visit(k, nil)
  223. if err != nil {
  224. return 0, err
  225. }
  226. vh, err := w.visit(v, nil)
  227. if err != nil {
  228. return 0, err
  229. }
  230. fieldHash := hashUpdateOrdered(w.h, kh, vh)
  231. h = hashUpdateUnordered(h, fieldHash)
  232. }
  233. if w.format != FormatV1 {
  234. // Important: read the docs for hashFinishUnordered
  235. h = hashFinishUnordered(w.h, h)
  236. }
  237. return h, nil
  238. case reflect.Struct:
  239. parent := v.Interface()
  240. var include Includable
  241. if impl, ok := parent.(Includable); ok {
  242. include = impl
  243. }
  244. if impl, ok := parent.(Hashable); ok {
  245. return impl.Hash()
  246. }
  247. // If we can address this value, check if the pointer value
  248. // implements our interfaces and use that if so.
  249. if v.CanAddr() {
  250. vptr := v.Addr()
  251. parentptr := vptr.Interface()
  252. if impl, ok := parentptr.(Includable); ok {
  253. include = impl
  254. }
  255. if impl, ok := parentptr.(Hashable); ok {
  256. return impl.Hash()
  257. }
  258. }
  259. t := v.Type()
  260. h, err := w.visit(reflect.ValueOf(t.Name()), nil)
  261. if err != nil {
  262. return 0, err
  263. }
  264. l := v.NumField()
  265. for i := 0; i < l; i++ {
  266. if innerV := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
  267. var f visitFlag
  268. fieldType := t.Field(i)
  269. if fieldType.PkgPath != "" {
  270. // Unexported
  271. continue
  272. }
  273. tag := fieldType.Tag.Get(w.tag)
  274. if tag == "ignore" || tag == "-" {
  275. // Ignore this field
  276. continue
  277. }
  278. if w.ignorezerovalue {
  279. if innerV.IsZero() {
  280. continue
  281. }
  282. }
  283. // if string is set, use the string value
  284. if tag == "string" || w.stringer {
  285. if impl, ok := innerV.Interface().(fmt.Stringer); ok {
  286. innerV = reflect.ValueOf(impl.String())
  287. } else if tag == "string" {
  288. // We only show this error if the tag explicitly
  289. // requests a stringer.
  290. return 0, &ErrNotStringer{
  291. Field: v.Type().Field(i).Name,
  292. }
  293. }
  294. }
  295. // Check if we implement includable and check it
  296. if include != nil {
  297. incl, err := include.HashInclude(fieldType.Name, innerV)
  298. if err != nil {
  299. return 0, err
  300. }
  301. if !incl {
  302. continue
  303. }
  304. }
  305. switch tag {
  306. case "set":
  307. f |= visitFlagSet
  308. }
  309. kh, err := w.visit(reflect.ValueOf(fieldType.Name), nil)
  310. if err != nil {
  311. return 0, err
  312. }
  313. vh, err := w.visit(innerV, &visitOpts{
  314. Flags: f,
  315. Struct: parent,
  316. StructField: fieldType.Name,
  317. })
  318. if err != nil {
  319. return 0, err
  320. }
  321. fieldHash := hashUpdateOrdered(w.h, kh, vh)
  322. h = hashUpdateUnordered(h, fieldHash)
  323. }
  324. if w.format != FormatV1 {
  325. // Important: read the docs for hashFinishUnordered
  326. h = hashFinishUnordered(w.h, h)
  327. }
  328. }
  329. return h, nil
  330. case reflect.Slice:
  331. // We have two behaviors here. If it isn't a set, then we just
  332. // visit all the elements. If it is a set, then we do a deterministic
  333. // hash code.
  334. var h uint64
  335. var set bool
  336. if opts != nil {
  337. set = (opts.Flags & visitFlagSet) != 0
  338. }
  339. l := v.Len()
  340. for i := 0; i < l; i++ {
  341. current, err := w.visit(v.Index(i), nil)
  342. if err != nil {
  343. return 0, err
  344. }
  345. if set || w.sets {
  346. h = hashUpdateUnordered(h, current)
  347. } else {
  348. h = hashUpdateOrdered(w.h, h, current)
  349. }
  350. }
  351. if set && w.format != FormatV1 {
  352. // Important: read the docs for hashFinishUnordered
  353. h = hashFinishUnordered(w.h, h)
  354. }
  355. return h, nil
  356. case reflect.String:
  357. // Directly hash
  358. w.h.Reset()
  359. _, err := w.h.Write([]byte(v.String()))
  360. return w.h.Sum64(), err
  361. default:
  362. return 0, fmt.Errorf("unknown kind to hash: %s", k)
  363. }
  364. }
  365. func hashUpdateOrdered(h hash.Hash64, a, b uint64) uint64 {
  366. // For ordered updates, use a real hash function
  367. h.Reset()
  368. // We just panic if the binary writes fail because we are writing
  369. // an int64 which should never be fail-able.
  370. e1 := binary.Write(h, binary.LittleEndian, a)
  371. e2 := binary.Write(h, binary.LittleEndian, b)
  372. if e1 != nil {
  373. panic(e1)
  374. }
  375. if e2 != nil {
  376. panic(e2)
  377. }
  378. return h.Sum64()
  379. }
  380. func hashUpdateUnordered(a, b uint64) uint64 {
  381. return a ^ b
  382. }
  383. // After mixing a group of unique hashes with hashUpdateUnordered, it's always
  384. // necessary to call hashFinishUnordered. Why? Because hashUpdateUnordered
  385. // is a simple XOR, and calling hashUpdateUnordered on hashes produced by
  386. // hashUpdateUnordered can effectively cancel out a previous change to the hash
  387. // result if the same hash value appears later on. For example, consider:
  388. //
  389. // hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) =
  390. // H("A") ^ H("B")) ^ (H("A") ^ H("C")) =
  391. // (H("A") ^ H("A")) ^ (H("B") ^ H(C)) =
  392. // H(B) ^ H(C) =
  393. // hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C"))
  394. //
  395. // hashFinishUnordered "hardens" the result, so that encountering partially
  396. // overlapping input data later on in a different context won't cancel out.
  397. func hashFinishUnordered(h hash.Hash64, a uint64) uint64 {
  398. h.Reset()
  399. // We just panic if the writes fail
  400. e1 := binary.Write(h, binary.LittleEndian, a)
  401. if e1 != nil {
  402. panic(e1)
  403. }
  404. return h.Sum64()
  405. }
  406. // visitFlag is used as a bitmask for affecting visit behavior
  407. type visitFlag uint
  408. const (
  409. visitFlagInvalid visitFlag = iota
  410. visitFlagSet = iota << 1
  411. )