Skip to content

Commit

Permalink
chore: add hashtriemap implementation
Browse files Browse the repository at this point in the history
This PR adds a hashtriemap implementation to the project. This is mostly a direct copy from unique package [^1]
internals [^2] with some adjustments to make it work with our project. Once `maphash.Comparable` lands we can remove
dependency on Go internals [^3]. Michael Knyszek says he plans to use this implementation as a general idea for the
future generic sync/v2.Map, but nothing is in stone yet.

[^1]: golang/go#62483
[^2]: https://go-review.googlesource.com/c/go/+/573956
[^3]: golang/go#54670

Signed-off-by: Dmitriy Matrenichev <dmitry.matrenichev@siderolabs.com>
  • Loading branch information
DmitriyMV committed May 27, 2024
1 parent 8485864 commit 7654108
Show file tree
Hide file tree
Showing 11 changed files with 1,144 additions and 37 deletions.
3 changes: 2 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-05-19T20:59:37Z by kres dccd292.
# Generated on 2024-05-27T12:37:27Z by kres b5844f8.

*
!channel
!concurrent
!containers
!ensure
!maps
Expand Down
38 changes: 27 additions & 11 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-05-19T18:27:53Z by kres dccd292.
# Generated on 2024-05-27T12:37:27Z by kres b5844f8.

name: default
concurrency:
Expand Down Expand Up @@ -29,16 +29,32 @@ jobs:
- self-hosted
- generic
if: (!startsWith(github.head_ref, 'renovate/') && !startsWith(github.head_ref, 'dependabot/'))
services:
buildkitd:
image: moby/buildkit:v0.13.2
options: --privileged
ports:
- 1234:1234
volumes:
- /var/lib/buildkit/${{ github.repository }}:/var/lib/buildkit
- /usr/etc/buildkit/buildkitd.toml:/etc/buildkit/buildkitd.toml
steps:
- name: gather-system-info
id: system-info
uses: kenchan0130/actions-system-info@v1.3.0
continue-on-error: true
- name: print-system-info
run: |
MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
OUTPUTS=(
"CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
"CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
"Hostname: ${{ steps.system-info.outputs.hostname }}"
"NodeName: ${NODE_NAME}"
"Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
"Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
"Name: ${{ steps.system-info.outputs.name }}"
"Platform: ${{ steps.system-info.outputs.platform }}"
"Release: ${{ steps.system-info.outputs.release }}"
"Total memory: ${MEMORY_GB} GB"
)
for OUTPUT in "${OUTPUTS[@]}";do
echo "${OUTPUT}"
done
continue-on-error: true
- name: checkout
uses: actions/checkout@v4
- name: Unshallow
Expand All @@ -49,7 +65,7 @@ jobs:
uses: docker/setup-buildx-action@v3
with:
driver: remote
endpoint: tcp://127.0.0.1:1234
endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
timeout-minutes: 10
- name: base
run: |
Expand Down
9 changes: 5 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,24 @@

# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-05-19T20:59:37Z by kres dccd292.
# Generated on 2024-05-27T23:26:26Z by kres bcb280a.

ARG TOOLCHAIN

# cleaned up specs and compiled versions
FROM scratch AS generate

# runs markdownlint
FROM docker.io/node:21.7.3-alpine3.19 AS lint-markdown
FROM docker.io/node:22.2.0-alpine3.19 AS lint-markdown
WORKDIR /src
RUN npm i -g markdownlint-cli@0.39.0
RUN npm i -g markdownlint-cli@0.41.0
RUN npm i sentences-per-line@0.2.1
COPY .markdownlint.json .
COPY ./README.md ./README.md
RUN markdownlint --ignore "CHANGELOG.md" --ignore "**/node_modules/**" --ignore '**/hack/chglog/**' --rules node_modules/sentences-per-line/index.js .

# base toolchain image
FROM ${TOOLCHAIN} AS toolchain
FROM --platform=${BUILDPLATFORM} ${TOOLCHAIN} AS toolchain
RUN apk --update --no-cache add bash curl build-base protoc protobuf-dev

# build tools
Expand Down Expand Up @@ -53,6 +53,7 @@ RUN cd .
RUN --mount=type=cache,target=/go/pkg go mod download
RUN --mount=type=cache,target=/go/pkg go mod verify
COPY ./channel ./channel
COPY ./concurrent ./concurrent
COPY ./containers ./containers
COPY ./ensure ./ensure
COPY ./maps ./maps
Expand Down
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-05-19T18:27:53Z by kres dccd292.
# Generated on 2024-05-27T23:26:26Z by kres bcb280a.

# common variables

Expand All @@ -17,13 +17,13 @@ WITH_RACE ?= false
REGISTRY ?= ghcr.io
USERNAME ?= siderolabs
REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME)
PROTOBUF_GO_VERSION ?= 1.33.0
PROTOBUF_GO_VERSION ?= 1.34.1
GRPC_GO_VERSION ?= 1.3.0
GRPC_GATEWAY_VERSION ?= 2.19.1
GRPC_GATEWAY_VERSION ?= 2.20.0
VTPROTOBUF_VERSION ?= 0.6.0
GOIMPORTS_VERSION ?= 0.21.0
DEEPCOPY_VERSION ?= v0.5.6
GOLANGCILINT_VERSION ?= v1.58.0
GOLANGCILINT_VERSION ?= v1.59.0
GOFUMPT_VERSION ?= v0.6.0
GO_VERSION ?= 1.22.3
GO_BUILDFLAGS ?=
Expand Down
74 changes: 74 additions & 0 deletions concurrent/go122.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build go1.22 && !go1.24 && !nomaptypehash

//nolint:revive,govet,stylecheck,nlreturn,wsl
package concurrent

import (
"math/rand/v2"
"unsafe"
)

// NewHashTrieMap creates a new HashTrieMap for the provided key and value.
func NewHashTrieMap[K, V comparable]() *HashTrieMap[K, V] {
var m map[K]V

mapType := efaceMapOf(m)
ht := &HashTrieMap[K, V]{
root: newIndirectNode[K, V](nil),
keyHash: mapType._type.Hasher,
seed: uintptr(rand.Uint64()),
}
return ht
}

// _MapType is runtime.maptype from runtime/type.go.
type _MapType struct {
_Type
Key *_Type
Elem *_Type
Bucket *_Type // internal type representing a hash bucket
// function for hashing keys (ptr to key, seed) -> hash
Hasher func(unsafe.Pointer, uintptr) uintptr
KeySize uint8 // size of key slot
ValueSize uint8 // size of elem slot
BucketSize uint16 // size of bucket
Flags uint32
}

// _Type is runtime._type from runtime/type.go.
type _Type struct {
Size_ uintptr
PtrBytes uintptr // number of (prefix) bytes in the type that can contain pointers
Hash uint32 // hash of type; avoids computation in hash tables
TFlag uint8 // extra type information flags
Align_ uint8 // alignment of variable with this type
FieldAlign_ uint8 // alignment of struct field with this type
Kind_ uint8 // enumeration for C
// function for comparing objects of this type
// (ptr to object A, ptr to object B) -> ==?
Equal func(unsafe.Pointer, unsafe.Pointer) bool
// GCData stores the GC type data for the garbage collector.
// If the KindGCProg bit is set in kind, GCData is a GC program.
// Otherwise it is a ptrmask bitmap. See mbitmap.go for details.
GCData *byte
Str int32 // string form
PtrToThis int32 // type for pointer to this type, may be zero
}

// efaceMap is runtime.eface from runtime/runtime2.go.
type efaceMap struct {
_type *_MapType
data unsafe.Pointer
}

func efaceMapOf(ep any) *efaceMap {
return (*efaceMap)(unsafe.Pointer(&ep))
}
71 changes: 71 additions & 0 deletions concurrent/go122_bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build go1.22 && !go1.24 && !nomaptypehash

//nolint:wsl,testpackage
package concurrent

import (
"testing"
)

func BenchmarkHashTrieMapLoadSmall(b *testing.B) {
benchmarkHashTrieMapLoad(b, testDataSmall[:])
}

func BenchmarkHashTrieMapLoad(b *testing.B) {
benchmarkHashTrieMapLoad(b, testData[:])
}

func BenchmarkHashTrieMapLoadLarge(b *testing.B) {
benchmarkHashTrieMapLoad(b, testDataLarge[:])
}

func benchmarkHashTrieMapLoad(b *testing.B, data []string) {
b.ReportAllocs()
m := NewHashTrieMap[string, int]()
for i := range data {
m.LoadOrStore(data[i], i)
}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
i := 0
for pb.Next() {
_, _ = m.Load(data[i])
i++
if i >= len(data) {
i = 0
}
}
})
}

func BenchmarkHashTrieMapLoadOrStore(b *testing.B) {
benchmarkHashTrieMapLoadOrStore(b, testData[:])
}

func BenchmarkHashTrieMapLoadOrStoreLarge(b *testing.B) {
benchmarkHashTrieMapLoadOrStore(b, testDataLarge[:])
}

func benchmarkHashTrieMapLoadOrStore(b *testing.B, data []string) {
b.ReportAllocs()
m := NewHashTrieMap[string, int]()

b.RunParallel(func(pb *testing.PB) {
i := 0
for pb.Next() {
_, _ = m.LoadOrStore(data[i], i)
i++
if i >= len(data) {
i = 0
}
}
})
}
35 changes: 35 additions & 0 deletions concurrent/go122_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build go1.22 && !go1.24 && !nomaptypehash

//nolint:nlreturn,wsl,testpackage
package concurrent

import (
"testing"
"unsafe"
)

func TestHashTrieMap(t *testing.T) {
testHashTrieMap(t, func() *HashTrieMap[string, int] {
return NewHashTrieMap[string, int]()
})
}

func TestHashTrieMapBadHash(t *testing.T) {
testHashTrieMap(t, func() *HashTrieMap[string, int] {
// Stub out the good hash function with a terrible one.
// Everything should still work as expected.
m := NewHashTrieMap[string, int]()
m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr {
return 0
}
return m
})
}
Loading

0 comments on commit 7654108

Please sign in to comment.