internal/sync: Add support for wide trie in sync.HashTrieMap-golang-AtomGit

已合并

internal/sync: Add support for wide trie in sync.HashTrieMap #51

spy20051623创建于 1月28日从spy20051623/golang： feat_hashtriemap 合入到 openeuler/golang： go1.24.6

internal/sync: Add support for wide trie in sync.HashTrieMap #51

已合并

spy20051623创建于 1月28日

从spy20051623/golang feat_hashtriemap合入到openeuler/golang go1.24.6

共 5 个文件变更+745-0

Asrc/internal/goexperiment/exp_widetrie_off.go+8-0

@@ -0,0 +1,8 @@
	1	+// Code generated by mkconsts.go. DO NOT EDIT.
	2	+
	3	+//go:build !goexperiment.widetrie
	4	+
	5	+package goexperiment
	6	+
	7	+const WideTrie = false
	8	+const WideTrieInt = 0

Asrc/internal/goexperiment/exp_widetrie_on.go+8-0

@@ -0,0 +1,8 @@
	1	+// Code generated by mkconsts.go. DO NOT EDIT.
	2	+
	3	+//go:build goexperiment.widetrie
	4	+
	5	+package goexperiment
	6	+
	7	+const WideTrie = true
	8	+const WideTrieInt = 1

Msrc/internal/goexperiment/flags.go+3-0

	@@ -128,4 +128,7 @@ type Flags struct {
128		128
129	// Synctest enables the testing/synctest package.	129	// Synctest enables the testing/synctest package.
130	Synctest bool	130	Synctest bool
		131	+
		132	+ // WideTrie uses larger children group in sync.hashtriemap
		133	+ WideTrie bool
131	}	134	}

Msrc/internal/sync/hashtriemap.go+1-0

	@@ -1,6 +1,7 @@
1	// Copyright 2024 The Go Authors. All rights reserved.	1	// Copyright 2024 The Go Authors. All rights reserved.
2	// Use of this source code is governed by a BSD-style	2	// Use of this source code is governed by a BSD-style
3	// license that can be found in the LICENSE file.	3	// license that can be found in the LICENSE file.
		4	+//go:build !goexperiment.widetrie
4		5
5	package sync	6	package sync
6		7

Asrc/internal/sync/hashtriemap_widetrie.go+725-0

@@ -0,0 +1,725 @@
	1	+// Copyright 2024 The Go Authors. All rights reserved.
	2	+// Use of this source code is governed by a BSD-style
	3	+// license that can be found in the LICENSE file.
	4	+//go:build goexperiment.widetrie
	5	+
	6	+package sync
	7	+
	8	+import (
	9	+ "internal/abi"
	10	+ "internal/goarch"
	11	+ "sync/atomic"
	12	+ "unsafe"
	13	+)
	14	+
	15	+// HashTrieMap is an implementation of a concurrent hash-trie. The implementation
	16	+// is designed around frequent loads, but offers decent performance for stores
	17	+// and deletes as well, especially if the map is larger. Its primary use-case is
	18	+// the unique package, but can be used elsewhere as well.
	19	+//
	20	+// The zero HashTrieMap is empty and ready to use.
	21	+// It must not be copied after first use.
	22	+type HashTrieMap[K comparable, V any] struct {
	23	+ inited atomic.Uint32
	24	+ initMu Mutex
	25	+ root atomic.Pointer[indirect[K, V]]
	26	+ keyHash hashFunc
	27	+ valEqual equalFunc
	28	+ seed uintptr
	29	+}
	30	+
	31	+func (ht *HashTrieMap[K, V]) init() {
	32	+ if ht.inited.Load() == 0 {
	33	+ ht.initSlow()
	34	+ }
	35	+}
	36	+
	37	+//go:noinline
	38	+func (ht *HashTrieMap[K, V]) initSlow() {
	39	+ ht.initMu.Lock()
	40	+ defer ht.initMu.Unlock()
	41	+
	42	+ if ht.inited.Load() != 0 {
	43	+ // Someone got to it while we were waiting.
	44	+ return
	45	+ }
	46	+
	47	+ // Set up root node, derive the hash function for the key, and the
	48	+ // equal function for the value, if any.
	49	+ var m map[K]V
	50	+ mapType := abi.TypeOf(m).MapType()
	51	+ ht.root.Store(newIndirectNode[K, V](nil))
	52	+ ht.keyHash = mapType.Hasher
	53	+ ht.valEqual = mapType.Elem.Equal
	54	+ ht.seed = uintptr(runtime_rand())
	55	+
	56	+ ht.inited.Store(1)
	57	+}
	58	+
	59	+type hashFunc func(unsafe.Pointer, uintptr) uintptr
	60	+type equalFunc func(unsafe.Pointer, unsafe.Pointer) bool
	61	+
	62	+// Load returns the value stored in the map for a key, or nil if no
	63	+// value is present.
	64	+// The ok result indicates whether value was found in the map.
	65	+func (ht *HashTrieMap[K, V]) Load(key K) (value V, ok bool) {
	66	+ ht.init()
	67	+ hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed)
	68	+
	69	+ i := ht.root.Load()
	70	+ hashShift := 8 * goarch.PtrSize
	71	+ for hashShift >= nChildrenLog2 {
	72	+ hashShift -= nChildrenLog2
	73	+
	74	+ n := i.children[(hash>>hashShift)&nChildrenMask].Load()
	75	+ if n == nil {
	76	+ return *new(V), false
	77	+ }
	78	+ if n.isEntry {
	79	+ return n.entry().lookup(key)
	80	+ }
	81	+ i = n.indirect()
	82	+ }
	83	+ panic("internal/sync.HashTrieMap: ran out of hash bits while iterating")
	84	+}
	85	+
	86	+// LoadOrStore returns the existing value for the key if present.
	87	+// Otherwise, it stores and returns the given value.
	88	+// The loaded result is true if the value was loaded, false if stored.
	89	+func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) {
	90	+ ht.init()
	91	+ hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed)
	92	+ var i *indirect[K, V]
	93	+ var hashShift uint
	94	+ var slot *atomic.Pointer[node[K, V]]
	95	+ var n *node[K, V]
	96	+ for {
	97	+ // Find the key or a candidate location for insertion.
	98	+ i = ht.root.Load()
	99	+ hashShift = 8 * goarch.PtrSize
	100	+ haveInsertPoint := false
	101	+ for hashShift >= nChildrenLog2 {
	102	+ hashShift -= nChildrenLog2
	103	+
	104	+ slot = &i.children[(hash>>hashShift)&nChildrenMask]
	105	+ n = slot.Load()
	106	+ if n == nil {
	107	+ // We found a nil slot which is a candidate for insertion.
	108	+ haveInsertPoint = true
	109	+ break
	110	+ }
	111	+ if n.isEntry {
	112	+ // We found an existing entry, which is as far as we can go.
	113	+ // If it stays this way, we'll have to replace it with an
	114	+ // indirect node.
	115	+ if v, ok := n.entry().lookup(key); ok {
	116	+ return v, true
	117	+ }
	118	+ haveInsertPoint = true
	119	+ break
	120	+ }
	121	+ i = n.indirect()
	122	+ }
	123	+ if !haveInsertPoint {
	124	+ panic("internal/sync.HashTrieMap: ran out of hash bits while iterating")
	125	+ }
	126	+
	127	+ // Grab the lock and double-check what we saw.
	128	+ i.mu.Lock()
	129	+ n = slot.Load()
	130	+ if (n == nil \|\| n.isEntry) && !i.dead.Load() {
	131	+ // What we saw is still true, so we can continue with the insert.
	132	+ break
	133	+ }
	134	+ // We have to start over.
	135	+ i.mu.Unlock()
	136	+ }
	137	+ // N.B. This lock is held from when we broke out of the outer loop above.
	138	+ // We specifically break this out so that we can use defer here safely.
	139	+ // One option is to break this out into a new function instead, but
	140	+ // there's so much local iteration state used below that this turns out
	141	+ // to be cleaner.
	142	+ defer i.mu.Unlock()
	143	+
	144	+ var oldEntry *entry[K, V]
	145	+ if n != nil {
	146	+ oldEntry = n.entry()
	147	+ if v, ok := oldEntry.lookup(key); ok {
	148	+ // Easy case: by loading again, it turns out exactly what we wanted is here!
	149	+ return v, true
	150	+ }
	151	+ }
	152	+ newEntry := newEntryNode(key, value)
	153	+ if oldEntry == nil {
	154	+ // Easy case: create a new entry and store it.
	155	+ slot.Store(&newEntry.node)
	156	+ } else {
	157	+ // We possibly need to expand the entry already there into one or more new nodes.
	158	+ //
	159	+ // Publish the node last, which will make both oldEntry and newEntry visible. We
	160	+ // don't want readers to be able to observe that oldEntry isn't in the tree.
	161	+ slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i))
	162	+ }
	163	+ return value, false
	164	+}
	165	+
	166	+// expand takes oldEntry and newEntry whose hashes conflict from bit 64 down to hashShift and
	167	+// produces a subtree of indirect nodes to hold the two new entries.
	168	+func (ht HashTrieMap[K, V]) expand(oldEntry, newEntry entry[K, V], newHash uintptr, hashShift uint, parent indirect[K, V]) node[K, V] {
	169	+ // Check for a hash collision.
	170	+ oldHash := ht.keyHash(unsafe.Pointer(&oldEntry.key), ht.seed)
	171	+ if oldHash == newHash {
	172	+ // Store the old entry in the new entry's overflow list, then store
	173	+ // the new entry.
	174	+ newEntry.overflow.Store(oldEntry)
	175	+ return &newEntry.node
	176	+ }
	177	+ // We have to add an indirect node. Worse still, we may need to add more than one.
	178	+ newIndirect := newIndirectNode(parent)
	179	+ top := newIndirect
	180	+ for {
	181	+ if hashShift < nChildrenLog2 {
	182	+ panic("internal/sync.HashTrieMap: ran out of hash bits while inserting")
	183	+ }
	184	+ hashShift -= nChildrenLog2 // hashShift is for the level parent is at. We need to go deeper.
	185	+ oi := (oldHash >> hashShift) & nChildrenMask
	186	+ ni := (newHash >> hashShift) & nChildrenMask
	187	+ if oi != ni {
	188	+ newIndirect.children[oi].Store(&oldEntry.node)
	189	+ newIndirect.children[ni].Store(&newEntry.node)
	190	+ break
	191	+ }
	192	+ nextIndirect := newIndirectNode(newIndirect)
	193	+ newIndirect.children[oi].Store(&nextIndirect.node)
	194	+ newIndirect = nextIndirect
	195	+ }
	196	+ return &top.node
	197	+}
	198	+
	199	+// Store sets the value for a key.
	200	+func (ht *HashTrieMap[K, V]) Store(key K, old V) {
	201	+ _, _ = ht.Swap(key, old)
	202	+}
	203	+
	204	+// Swap swaps the value for a key and returns the previous value if any.
	205	+// The loaded result reports whether the key was present.
	206	+func (ht *HashTrieMap[K, V]) Swap(key K, new V) (previous V, loaded bool) {
	207	+ ht.init()
	208	+ hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed)
	209	+ var i *indirect[K, V]
	210	+ var hashShift uint
	211	+ var slot *atomic.Pointer[node[K, V]]
	212	+ var n *node[K, V]
	213	+ for {
	214	+ // Find the key or a candidate location for insertion.
	215	+ i = ht.root.Load()
	216	+ hashShift = 8 * goarch.PtrSize
	217	+ haveInsertPoint := false
	218	+ for hashShift >= nChildrenLog2 {
	219	+ hashShift -= nChildrenLog2
	220	+
	221	+ slot = &i.children[(hash>>hashShift)&nChildrenMask]
	222	+ n = slot.Load()
	223	+ if n == nil \|\| n.isEntry {
	224	+ // We found a nil slot which is a candidate for insertion,
	225	+ // or an existing entry that we'll replace.
	226	+ haveInsertPoint = true
	227	+ break
	228	+ }
	229	+ i = n.indirect()
	230	+ }
	231	+ if !haveInsertPoint {
	232	+ panic("internal/sync.HashTrieMap: ran out of hash bits while iterating")
	233	+ }
	234	+
	235	+ // Grab the lock and double-check what we saw.
	236	+ i.mu.Lock()
	237	+ n = slot.Load()
	238	+ if (n == nil \|\| n.isEntry) && !i.dead.Load() {
	239	+ // What we saw is still true, so we can continue with the insert.
	240	+ break
	241	+ }
	242	+ // We have to start over.
	243	+ i.mu.Unlock()
	244	+ }
	245	+ // N.B. This lock is held from when we broke out of the outer loop above.
	246	+ // We specifically break this out so that we can use defer here safely.
	247	+ // One option is to break this out into a new function instead, but
	248	+ // there's so much local iteration state used below that this turns out
	249	+ // to be cleaner.
	250	+ defer i.mu.Unlock()
	251	+
	252	+ var zero V
	253	+ var oldEntry *entry[K, V]
	254	+ if n != nil {
	255	+ // Swap if the keys compare.
	256	+ oldEntry = n.entry()
	257	+ newEntry, old, swapped := oldEntry.swap(key, new)
	258	+ if swapped {
	259	+ slot.Store(&newEntry.node)
	260	+ return old, true
	261	+ }
	262	+ }
	263	+ // The keys didn't compare, so we're doing an insertion.
	264	+ newEntry := newEntryNode(key, new)
	265	+ if oldEntry == nil {
	266	+ // Easy case: create a new entry and store it.
	267	+ slot.Store(&newEntry.node)
	268	+ } else {
	269	+ // We possibly need to expand the entry already there into one or more new nodes.
	270	+ //
	271	+ // Publish the node last, which will make both oldEntry and newEntry visible. We
	272	+ // don't want readers to be able to observe that oldEntry isn't in the tree.
	273	+ slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i))
	274	+ }
	275	+ return zero, false
	276	+}
	277	+
	278	+// CompareAndSwap swaps the old and new values for key
	279	+// if the value stored in the map is equal to old.
	280	+// The value type must be of a comparable type, otherwise CompareAndSwap will panic.
	281	+func (ht *HashTrieMap[K, V]) CompareAndSwap(key K, old, new V) (swapped bool) {
	282	+ ht.init()
	283	+ if ht.valEqual == nil {
	284	+ panic("called CompareAndSwap when value is not of comparable type")
	285	+ }
	286	+ hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed)
	287	+
	288	+ // Find a node with the key and compare with it. n != nil if we found the node.
	289	+ i, _, slot, n := ht.find(key, hash, ht.valEqual, old)
	290	+ if i != nil {
	291	+ defer i.mu.Unlock()
	292	+ }
	293	+ if n == nil {
	294	+ return false
	295	+ }
	296	+
	297	+ // Try to swap the entry.
	298	+ e, swapped := n.entry().compareAndSwap(key, old, new, ht.valEqual)
	299	+ if !swapped {
	300	+ // Nothing was actually swapped, which means the node is no longer there.
	301	+ return false
	302	+ }
	303	+ // Store the entry back because it changed.
	304	+ slot.Store(&e.node)
	305	+ return true
	306	+}
	307	+
	308	+// LoadAndDelete deletes the value for a key, returning the previous value if any.
	309	+// The loaded result reports whether the key was present.
	310	+func (ht *HashTrieMap[K, V]) LoadAndDelete(key K) (value V, loaded bool) {
	311	+ ht.init()
	312	+ hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed)
	313	+
	314	+ // Find a node with the key and compare with it. n != nil if we found the node.
	315	+ i, hashShift, slot, n := ht.find(key, hash, nil, *new(V))
	316	+ if n == nil {
	317	+ if i != nil {
	318	+ i.mu.Unlock()
	319	+ }
	320	+ return *new(V), false
	321	+ }
	322	+
	323	+ // Try to delete the entry.
	324	+ v, e, loaded := n.entry().loadAndDelete(key)
	325	+ if !loaded {
	326	+ // Nothing was actually deleted, which means the node is no longer there.
	327	+ i.mu.Unlock()
	328	+ return *new(V), false
	329	+ }
	330	+ if e != nil {
	331	+ // We didn't actually delete the whole entry, just one entry in the chain.
	332	+ // Nothing else to do, since the parent is definitely not empty.
	333	+ slot.Store(&e.node)
	334	+ i.mu.Unlock()
	335	+ return v, true
	336	+ }
	337	+ // Delete the entry.
	338	+ slot.Store(nil)
	339	+
	340	+ // Check if the node is now empty (and isn't the root), and delete it if able.
	341	+ for i.parent != nil && i.empty() {
	342	+ if hashShift == 8*goarch.PtrSize {
	343	+ panic("internal/sync.HashTrieMap: ran out of hash bits while iterating")
	344	+ }
	345	+ hashShift += nChildrenLog2
	346	+
	347	+ // Delete the current node in the parent.
	348	+ parent := i.parent
	349	+ parent.mu.Lock()
	350	+ i.dead.Store(true)
	351	+ parent.children[(hash>>hashShift)&nChildrenMask].Store(nil)
	352	+ i.mu.Unlock()
	353	+ i = parent
	354	+ }
	355	+ i.mu.Unlock()
	356	+ return v, true
	357	+}
	358	+
	359	+// Delete deletes the value for a key.
	360	+func (ht *HashTrieMap[K, V]) Delete(key K) {
	361	+ _, _ = ht.LoadAndDelete(key)
	362	+}
	363	+
	364	+// CompareAndDelete deletes the entry for key if its value is equal to old.
	365	+// The value type must be comparable, otherwise this CompareAndDelete will panic.
	366	+//
	367	+// If there is no current value for key in the map, CompareAndDelete returns false
	368	+// (even if the old value is the nil interface value).
	369	+func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) {
	370	+ ht.init()
	371	+ if ht.valEqual == nil {
	372	+ panic("called CompareAndDelete when value is not of comparable type")
	373	+ }
	374	+ hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed)
	375	+
	376	+ // Find a node with the key. n != nil if we found the node.
	377	+ i, hashShift, slot, n := ht.find(key, hash, nil, *new(V))
	378	+ if n == nil {
	379	+ if i != nil {
	380	+ i.mu.Unlock()
	381	+ }
	382	+ return false
	383	+ }
	384	+
	385	+ // Try to delete the entry.
	386	+ e, deleted := n.entry().compareAndDelete(key, old, ht.valEqual)
	387	+ if !deleted {
	388	+ // Nothing was actually deleted, which means the node is no longer there.
	389	+ i.mu.Unlock()
	390	+ return false
	391	+ }
	392	+ if e != nil {
	393	+ // We didn't actually delete the whole entry, just one entry in the chain.
	394	+ // Nothing else to do, since the parent is definitely not empty.
	395	+ slot.Store(&e.node)
	396	+ i.mu.Unlock()
	397	+ return true
	398	+ }
	399	+ // Delete the entry.
	400	+ slot.Store(nil)
	401	+
	402	+ // Check if the node is now empty (and isn't the root), and delete it if able.
	403	+ for i.parent != nil && i.empty() {
	404	+ if hashShift == 8*goarch.PtrSize {
	405	+ panic("internal/sync.HashTrieMap: ran out of hash bits while iterating")
	406	+ }
	407	+ hashShift += nChildrenLog2
	408	+
	409	+ // Delete the current node in the parent.
	410	+ parent := i.parent
	411	+ parent.mu.Lock()
	412	+ i.dead.Store(true)
	413	+ parent.children[(hash>>hashShift)&nChildrenMask].Store(nil)
	414	+ i.mu.Unlock()
	415	+ i = parent
	416	+ }
	417	+ i.mu.Unlock()
	418	+ return true
	419	+}
	420	+
	421	+// find searches the tree for a node that contains key (hash must be the hash of key).
	422	+// If valEqual != nil, then it will also enforce that the values are equal as well.
	423	+//
	424	+// Returns a non-nil node, which will always be an entry, if found.
	425	+//
	426	+// If i != nil then i.mu is locked, and it is the caller's responsibility to unlock it.
	427	+func (ht HashTrieMap[K, V]) find(key K, hash uintptr, valEqual equalFunc, value V) (i indirect[K, V], hashShift uint, slot atomic.Pointer[node[K, V]], n node[K, V]) {
	428	+ for {
	429	+ // Find the key or return if it's not there.
	430	+ i = ht.root.Load()
	431	+ hashShift = 8 * goarch.PtrSize
	432	+ found := false
	433	+ for hashShift >= nChildrenLog2 {
	434	+ hashShift -= nChildrenLog2
	435	+
	436	+ slot = &i.children[(hash>>hashShift)&nChildrenMask]
	437	+ n = slot.Load()
	438	+ if n == nil {
	439	+ // Nothing to compare with. Give up.
	440	+ i = nil
	441	+ return
	442	+ }
	443	+ if n.isEntry {
	444	+ // We found an entry. Check if it matches.
	445	+ if _, ok := n.entry().lookupWithValue(key, value, valEqual); !ok {
	446	+ // No match, comparison failed.
	447	+ i = nil
	448	+ n = nil
	449	+ return
	450	+ }
	451	+ // We've got a match. Prepare to perform an operation on the key.
	452	+ found = true
	453	+ break
	454	+ }
	455	+ i = n.indirect()
	456	+ }
	457	+ if !found {
	458	+ panic("internal/sync.HashTrieMap: ran out of hash bits while iterating")
	459	+ }
	460	+
	461	+ // Grab the lock and double-check what we saw.
	462	+ i.mu.Lock()
	463	+ n = slot.Load()
	464	+ if !i.dead.Load() && (n == nil \|\| n.isEntry) {
	465	+ // Either we've got a valid node or the node is now nil under the lock.
	466	+ // In either case, we're done here.
	467	+ return
	468	+ }
	469	+ // We have to start over.
	470	+ i.mu.Unlock()
	471	+ }
	472	+}
	473	+
	474	+// All returns an iterator over each key and value present in the map.
	475	+//
	476	+// The iterator does not necessarily correspond to any consistent snapshot of the
	477	+// HashTrieMap's contents: no key will be visited more than once, but if the value
	478	+// for any key is stored or deleted concurrently (including by yield), the iterator
	479	+// may reflect any mapping for that key from any point during iteration. The iterator
	480	+// does not block other methods on the receiver; even yield itself may call any
	481	+// method on the HashTrieMap.
	482	+func (ht *HashTrieMap[K, V]) All() func(yield func(K, V) bool) {
	483	+ ht.init()
	484	+ return func(yield func(key K, value V) bool) {
	485	+ ht.iter(ht.root.Load(), yield)
	486	+ }
	487	+}
	488	+
	489	+// Range calls f sequentially for each key and value present in the map.
	490	+// If f returns false, range stops the iteration.
	491	+//
	492	+// This exists for compatibility with sync.Map; All should be preferred.
	493	+// It provides the same guarantees as sync.Map, and All.
	494	+func (ht *HashTrieMap[K, V]) Range(yield func(K, V) bool) {
	495	+ ht.init()
	496	+ ht.iter(ht.root.Load(), yield)
	497	+}
	498	+
	499	+func (ht HashTrieMap[K, V]) iter(i indirect[K, V], yield func(key K, value V) bool) bool {
	500	+ for j := range i.children {
	501	+ n := i.children[j].Load()
	502	+ if n == nil {
	503	+ continue
	504	+ }
	505	+ if !n.isEntry {
	506	+ if !ht.iter(n.indirect(), yield) {
	507	+ return false
	508	+ }
	509	+ continue
	510	+ }
	511	+ e := n.entry()
	512	+ for e != nil {
	513	+ if !yield(e.key, e.value) {
	514	+ return false
	515	+ }
	516	+ e = e.overflow.Load()
	517	+ }
	518	+ }
	519	+ return true
	520	+}
	521	+
	522	+// Clear deletes all the entries, resulting in an empty HashTrieMap.
	523	+func (ht *HashTrieMap[K, V]) Clear() {
	524	+ ht.init()
	525	+
	526	+ // It's sufficient to just drop the root on the floor, but the root
	527	+ // must always be non-nil.
	528	+ ht.root.Store(newIndirectNode[K, V](nil))
	529	+}
	530	+
	531	+const (
	532	+ // 16 children. This seems to be the sweet spot for
	533	+ // load performance: any smaller and we lose out on
	534	+ // 50% or more in CPU performance. Any larger and the
	535	+ // returns are minuscule (~1% improvement for 32 children).
	536	+ nChildrenLog2 = 7
	537	+ nChildren = 1 << nChildrenLog2
	538	+ nChildrenMask = nChildren - 1
	539	+)
	540	+
	541	+// indirect is an internal node in the hash-trie.
	542	+type indirect[K comparable, V any] struct {
	543	+ node[K, V]
	544	+ dead atomic.Bool
	545	+ mu Mutex // Protects mutation to children and any children that are entry nodes.
	546	+ parent *indirect[K, V]
	547	+ children [nChildren]atomic.Pointer[node[K, V]]
	548	+}
	549	+
	550	+func newIndirectNode[K comparable, V any](parent indirect[K, V]) indirect[K, V] {
	551	+ return &indirect[K, V]{node: node[K, V]{isEntry: false}, parent: parent}
	552	+}
	553	+
	554	+func (i *indirect[K, V]) empty() bool {
	555	+ nc := 0
	556	+ for j := range i.children {
	557	+ if i.children[j].Load() != nil {
	558	+ nc++
	559	+ }
	560	+ }
	561	+ return nc == 0
	562	+}
	563	+
	564	+// entry is a leaf node in the hash-trie.
	565	+type entry[K comparable, V any] struct {
	566	+ node[K, V]
	567	+ overflow atomic.Pointer[entry[K, V]] // Overflow for hash collisions.
	568	+ key K
	569	+ value V
	570	+}
	571	+
	572	+func newEntryNode[K comparable, V any](key K, value V) *entry[K, V] {
	573	+ return &entry[K, V]{
	574	+ node: node[K, V]{isEntry: true},
	575	+ key: key,
	576	+ value: value,
	577	+ }
	578	+}
	579	+
	580	+func (e *entry[K, V]) lookup(key K) (V, bool) {
	581	+ for e != nil {
	582	+ if e.key == key {
	583	+ return e.value, true
	584	+ }
	585	+ e = e.overflow.Load()
	586	+ }
	587	+ return *new(V), false
	588	+}
	589	+
	590	+func (e *entry[K, V]) lookupWithValue(key K, value V, valEqual equalFunc) (V, bool) {
	591	+ for e != nil {
	592	+ if e.key == key && (valEqual == nil \|\| valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value)))) {
	593	+ return e.value, true
	594	+ }
	595	+ e = e.overflow.Load()
	596	+ }
	597	+ return *new(V), false
	598	+}
	599	+
	600	+// swap replaces an entry in the overflow chain if keys compare equal. Returns the new entry chain,
	601	+// the old value, and whether or not anything was swapped.
	602	+//
	603	+// swap must be called under the mutex of the indirect node which e is a child of.
	604	+func (head entry[K, V]) swap(key K, new V) (entry[K, V], V, bool) {
	605	+ if head.key == key {
	606	+ // Return the new head of the list.
	607	+ e := newEntryNode(key, new)
	608	+ if chain := head.overflow.Load(); chain != nil {
	609	+ e.overflow.Store(chain)
	610	+ }
	611	+ return e, head.value, true
	612	+ }
	613	+ i := &head.overflow
	614	+ e := i.Load()
	615	+ for e != nil {
	616	+ if e.key == key {
	617	+ eNew := newEntryNode(key, new)
	618	+ eNew.overflow.Store(e.overflow.Load())
	619	+ i.Store(eNew)
	620	+ return head, e.value, true
	621	+ }
	622	+ i = &e.overflow
	623	+ e = e.overflow.Load()
	624	+ }
	625	+ var zero V
	626	+ return head, zero, false
	627	+}
	628	+
	629	+// compareAndSwap replaces an entry in the overflow chain if both the key and value compare
	630	+// equal. Returns the new entry chain and whether or not anything was swapped.
	631	+//
	632	+// compareAndSwap must be called under the mutex of the indirect node which e is a child of.
	633	+func (head entry[K, V]) compareAndSwap(key K, old, new V, valEqual equalFunc) (entry[K, V], bool) {
	634	+ if head.key == key && valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&old))) {
	635	+ // Return the new head of the list.
	636	+ e := newEntryNode(key, new)
	637	+ if chain := head.overflow.Load(); chain != nil {
	638	+ e.overflow.Store(chain)
	639	+ }
	640	+ return e, true
	641	+ }
	642	+ i := &head.overflow
	643	+ e := i.Load()
	644	+ for e != nil {
	645	+ if e.key == key && valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&old))) {
	646	+ eNew := newEntryNode(key, new)
	647	+ eNew.overflow.Store(e.overflow.Load())
	648	+ i.Store(eNew)
	649	+ return head, true
	650	+ }
	651	+ i = &e.overflow
	652	+ e = e.overflow.Load()
	653	+ }
	654	+ return head, false
	655	+}
	656	+
	657	+// loadAndDelete deletes an entry in the overflow chain by key. Returns the value for the key, the new
	658	+// entry chain and whether or not anything was loaded (and deleted).
	659	+//
	660	+// loadAndDelete must be called under the mutex of the indirect node which e is a child of.
	661	+func (head entry[K, V]) loadAndDelete(key K) (V, entry[K, V], bool) {
	662	+ if head.key == key {
	663	+ // Drop the head of the list.
	664	+ return head.value, head.overflow.Load(), true
	665	+ }
	666	+ i := &head.overflow
	667	+ e := i.Load()
	668	+ for e != nil {
	669	+ if e.key == key {
	670	+ i.Store(e.overflow.Load())
	671	+ return e.value, head, true
	672	+ }
	673	+ i = &e.overflow
	674	+ e = e.overflow.Load()
	675	+ }
	676	+ return *new(V), head, false
	677	+}
	678	+
	679	+// compareAndDelete deletes an entry in the overflow chain if both the key and value compare
	680	+// equal. Returns the new entry chain and whether or not anything was deleted.
	681	+//
	682	+// compareAndDelete must be called under the mutex of the indirect node which e is a child of.
	683	+func (head entry[K, V]) compareAndDelete(key K, value V, valEqual equalFunc) (entry[K, V], bool) {
	684	+ if head.key == key && valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&value))) {
	685	+ // Drop the head of the list.
	686	+ return head.overflow.Load(), true
	687	+ }
	688	+ i := &head.overflow
	689	+ e := i.Load()
	690	+ for e != nil {
	691	+ if e.key == key && valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value))) {
	692	+ i.Store(e.overflow.Load())
	693	+ return head, true
	694	+ }
	695	+ i = &e.overflow
	696	+ e = e.overflow.Load()
	697	+ }
	698	+ return head, false
	699	+}
	700	+
	701	+// node is the header for a node. It's polymorphic and
	702	+// is actually either an entry or an indirect.
	703	+type node[K comparable, V any] struct {
	704	+ isEntry bool
	705	+}
	706	+
	707	+func (n node[K, V]) entry() entry[K, V] {
	708	+ if !n.isEntry {
	709	+ panic("called entry on non-entry node")
	710	+ }
	711	+ return (*entry[K, V])(unsafe.Pointer(n))
	712	+}
	713	+
	714	+func (n node[K, V]) indirect() indirect[K, V] {
	715	+ if n.isEntry {
	716	+ panic("called indirect on entry node")
	717	+ }
	718	+ return (*indirect[K, V])(unsafe.Pointer(n))
	719	+}
	720	+
	721	+// Pull in runtime.rand so that we don't need to take a dependency
	722	+// on math/rand/v2.
	723	+//
	724	+//go:linkname runtime_rand runtime.rand
	725	+func runtime_rand() uint64