* Copyright (c) 2024 Huawei Technologies Co., Ltd.
* openFuyao is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
*/
package blockkey
import (
"reflect"
"testing"
"github.com/go-logr/logr/testr"
"gitcode.com/openFuyao/cache-indexer/pkg/apis"
)
const (
testSeed = "12345"
testBlockSize = int64(4)
testSmallBlockSize = int64(2)
testTokenCount = 16
testShortTokenCount = 3
)
func TestNew_RequiresSeed(t *testing.T) {
if _, err := New(testr.New(t), Config{}); err == nil {
t.Fatal("expected error for empty PYTHONHASHSEED")
}
}
func TestNew_WithValidConfig(t *testing.T) {
b, err := New(testr.New(t), Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: true})
if err != nil {
t.Fatalf("New: %v", err)
}
if b == nil {
t.Fatal("expected non-nil builder")
}
}
func TestNew_WithXXHash3Algo(t *testing.T) {
b, err := New(testr.New(t), Config{PythonHashSeed: testSeed, Algo: HashXXHash3CBOR, EmitInt64: true})
if err != nil {
t.Fatalf("New: %v", err)
}
tokens := []int32{10, 20, 30, 40}
got, err := b.BuildFromRequest(apis.ModelContext{BlockSize: testBlockSize}, tokens)
if err != nil {
t.Fatalf("build: %v", err)
}
if len(got) != 1 {
t.Fatalf("expected 1 block, got %d", len(got))
}
}
func TestNew_DisableInt64Hash(t *testing.T) {
b, err := New(testr.New(t), Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: false})
if err != nil {
t.Fatalf("New: %v", err)
}
got, err := b.BuildFromRequest(apis.ModelContext{BlockSize: testBlockSize}, []int32{10, 20, 30, 40})
if err != nil {
t.Fatalf("build: %v", err)
}
if len(got[0]) != 64 {
t.Fatalf("expected full hex (64 chars), got %d", len(got[0]))
}
}
func TestBuildFromRequest_BlockSizeValidation(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, EmitInt64: true})
if _, err := b.BuildFromRequest(apis.ModelContext{BlockSize: 0}, []int32{1, 2}); err == nil {
t.Fatal("expected error for block_size=0")
}
}
func TestBuildFromRequest_ShorterThanOneBlock_ReturnsNil(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, EmitInt64: true})
out, err := b.BuildFromRequest(apis.ModelContext{BlockSize: testBlockSize}, []int32{1, 2, testShortTokenCount})
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if out != nil {
t.Fatalf("expected nil, got %v", out)
}
}
func TestSHA256CBOR_GoldenChain_NoSalt(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: true})
tokens := []int32{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160}
got, err := b.BuildFromRequest(apis.ModelContext{BlockSize: testBlockSize}, tokens)
if err != nil {
t.Fatalf("build: %v", err)
}
want := []apis.BlockHash{
"32bafc903fcc3d41",
"85561c9d5b82a733",
"ad51afb517a4a0d5",
"a4bacdb3459e9fcd",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("sha256 chain mismatch\n got=%v\nwant=%v", got, want)
}
}
func TestSHA256CBOR_GoldenChain_WithCacheSalt(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: true})
got, err := b.BuildFromRequest(
apis.ModelContext{BlockSize: testSmallBlockSize, CacheSalt: "abc"},
[]int32{1, 2, 3, 4, 5, 6},
)
if err != nil {
t.Fatalf("build: %v", err)
}
want := []apis.BlockHash{
"2aa1822e9d32e3f9",
"bf212bcd6f43c610",
"f13afb3eddfc87cf",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("salt chain mismatch\n got=%v\nwant=%v", got, want)
}
}
func TestXXHash3CBOR_GoldenChain(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashXXHash3CBOR, EmitInt64: true})
tokens := []int32{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160}
got, err := b.BuildFromRequest(apis.ModelContext{BlockSize: testBlockSize}, tokens)
if err != nil {
t.Fatalf("build: %v", err)
}
want := []apis.BlockHash{
"962478845458fd72",
"b52a3f6d44df5af0",
"ad1aaaf27d07b4ce",
"9b69e0e949610060",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("xxh3 chain mismatch\n got=%v\nwant=%v", got, want)
}
}
func TestEmitFullHexMode(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: false})
got, err := b.BuildFromRequest(
apis.ModelContext{BlockSize: testBlockSize},
[]int32{10, 20, 30, 40},
)
if err != nil {
t.Fatalf("build: %v", err)
}
if len(got) != 1 {
t.Fatalf("want 1 block, got %d", len(got))
}
const wantFull = "9bf368022e49912ba16c481c749a6a55b3117e0ceacb09ae32bafc903fcc3d41"
if string(got[0]) != wantFull {
t.Fatalf("full hex mismatch\n got=%s\nwant=%s", got[0], wantFull)
}
}
func TestEncodeBlockHash_ShortHash(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: true})
builder := b.(*builder)
shortHash := []byte{0x01, 0x02, 0x03, 0x04}
result := builder.encodeBlockHash(shortHash)
if len(result) != 16 {
t.Errorf("expected 16-char hex, got %d chars: %s", len(result), result)
}
expected := "0000000001020304"
if string(result) != expected {
t.Errorf("expected %s, got %s", expected, result)
}
oneByteHash := []byte{0xff}
result = builder.encodeBlockHash(oneByteHash)
if len(result) != 16 {
t.Errorf("expected 16-char hex for 1-byte, got %d", len(result))
}
expectedOne := "00000000000000ff"
if string(result) != expectedOne {
t.Errorf("expected %s, got %s", expectedOne, result)
}
}
func TestBuildFromRequest_EmptyTokenList(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, EmitInt64: true})
got, err := b.BuildFromRequest(apis.ModelContext{BlockSize: testBlockSize}, []int32{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != nil {
t.Errorf("expected nil for empty tokens, got %v", got)
}
}
func TestBuildFromRequest_MultipleBlocks(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: true})
tokens := make([]int32, 32)
for i := 0; i < 32; i++ {
tokens[i] = int32(i + 1)
}
got, err := b.BuildFromRequest(apis.ModelContext{BlockSize: testBlockSize}, tokens)
if err != nil {
t.Fatalf("build: %v", err)
}
if len(got) != 8 {
t.Fatalf("expected 8 blocks, got %d", len(got))
}
for i, hash := range got {
if len(hash) != 16 {
t.Errorf("block %d: expected 16-char hash, got %d", i, len(hash))
}
}
}
func TestBuildFromRequest_NegativeBlockSize(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, EmitInt64: true})
_, err := b.BuildFromRequest(apis.ModelContext{BlockSize: -1}, []int32{1, 2, 3, 4})
if err == nil {
t.Fatal("expected error for negative block_size")
}
}
func TestNoneHashBytes_CalledOnce(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashSHA256CBOR, EmitInt64: true})
builder := b.(*builder)
hash1, err1 := builder.noneHashBytes()
if err1 != nil {
t.Fatalf("first call failed: %v", err1)
}
hash2, err2 := builder.noneHashBytes()
if err2 != nil {
t.Fatalf("second call failed: %v", err2)
}
if hash1 == nil || hash2 == nil {
t.Fatal("hash should not be nil")
}
if !reflect.DeepEqual(hash1, hash2) {
t.Errorf("hash should be identical across calls: %v vs %v", hash1, hash2)
}
if len(hash1) != 32 {
t.Errorf("sha256 hash should be 32 bytes, got %d", len(hash1))
}
}
func TestNoneHashBytes_XXHash3(t *testing.T) {
b := mustBuilder(t, Config{PythonHashSeed: testSeed, Algo: HashXXHash3CBOR, EmitInt64: true})
builder := b.(*builder)
hash, err := builder.noneHashBytes()
if err != nil {
t.Fatalf("noneHashBytes: %v", err)
}
if len(hash) != 16 {
t.Errorf("xxh3 hash should be 16 bytes, got %d", len(hash))
}
}
func mustBuilder(t *testing.T, cfg Config) Builder {
t.Helper()
b, err := New(testr.New(t), cfg)
if err != nil {
t.Fatalf("New: %v", err)
}
return b
}