package tokenizerv1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
)
const (
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type TokenizerProvider int32
const (
TokenizerProvider_TOKENIZER_PROVIDER_UNSPECIFIED TokenizerProvider = 0
TokenizerProvider_TOKENIZER_PROVIDER_AUTO TokenizerProvider = 1
TokenizerProvider_TOKENIZER_PROVIDER_HUGGINGFACE TokenizerProvider = 2
TokenizerProvider_TOKENIZER_PROVIDER_MODELSCOPE TokenizerProvider = 3
)
var (
TokenizerProvider_name = map[int32]string{
0: "TOKENIZER_PROVIDER_UNSPECIFIED",
1: "TOKENIZER_PROVIDER_AUTO",
2: "TOKENIZER_PROVIDER_HUGGINGFACE",
3: "TOKENIZER_PROVIDER_MODELSCOPE",
}
TokenizerProvider_value = map[string]int32{
"TOKENIZER_PROVIDER_UNSPECIFIED": 0,
"TOKENIZER_PROVIDER_AUTO": 1,
"TOKENIZER_PROVIDER_HUGGINGFACE": 2,
"TOKENIZER_PROVIDER_MODELSCOPE": 3,
}
)
func (x TokenizerProvider) Enum() *TokenizerProvider {
p := new(TokenizerProvider)
*p = x
return p
}
func (x TokenizerProvider) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (TokenizerProvider) Descriptor() protoreflect.EnumDescriptor {
return file_api_tokenizer_v1_tokenizer_proto_enumTypes[0].Descriptor()
}
func (TokenizerProvider) Type() protoreflect.EnumType {
return &file_api_tokenizer_v1_tokenizer_proto_enumTypes[0]
}
func (x TokenizerProvider) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
func (TokenizerProvider) EnumDescriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{0}
}
type Modality int32
const (
Modality_MODALITY_UNSPECIFIED Modality = 0
Modality_MODALITY_IMAGE Modality = 1
)
var (
Modality_name = map[int32]string{
0: "MODALITY_UNSPECIFIED",
1: "MODALITY_IMAGE",
}
Modality_value = map[string]int32{
"MODALITY_UNSPECIFIED": 0,
"MODALITY_IMAGE": 1,
}
)
func (x Modality) Enum() *Modality {
p := new(Modality)
*p = x
return p
}
func (x Modality) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (Modality) Descriptor() protoreflect.EnumDescriptor {
return file_api_tokenizer_v1_tokenizer_proto_enumTypes[1].Descriptor()
}
func (Modality) Type() protoreflect.EnumType {
return &file_api_tokenizer_v1_tokenizer_proto_enumTypes[1]
}
func (x Modality) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
func (Modality) EnumDescriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{1}
}
type MultiModalFeature struct {
state protoimpl.MessageState `protogen:"open.v1"`
Modality Modality `protobuf:"varint,1,opt,name=modality,proto3,enum=hermes.tokenizer.v1.Modality" json:"modality,omitempty"`
Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"`
Offset int32 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"`
Length int32 `protobuf:"varint,4,opt,name=length,proto3" json:"length,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *MultiModalFeature) Reset() {
*x = MultiModalFeature{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *MultiModalFeature) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MultiModalFeature) ProtoMessage() {}
func (x *MultiModalFeature) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[0]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*MultiModalFeature) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{0}
}
func (x *MultiModalFeature) GetModality() Modality {
if x != nil {
return x.Modality
}
return Modality_MODALITY_UNSPECIFIED
}
func (x *MultiModalFeature) GetHash() string {
if x != nil {
return x.Hash
}
return ""
}
func (x *MultiModalFeature) GetOffset() int32 {
if x != nil {
return x.Offset
}
return 0
}
func (x *MultiModalFeature) GetLength() int32 {
if x != nil {
return x.Length
}
return 0
}
type TokenizationResult struct {
state protoimpl.MessageState `protogen:"open.v1"`
TokenIds []uint32 `protobuf:"varint,1,rep,packed,name=token_ids,json=tokenIds,proto3" json:"token_ids,omitempty"`
MultimodalFeatures []*MultiModalFeature `protobuf:"bytes,2,rep,name=multimodal_features,json=multimodalFeatures,proto3" json:"multimodal_features,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *TokenizationResult) Reset() {
*x = TokenizationResult{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *TokenizationResult) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*TokenizationResult) ProtoMessage() {}
func (x *TokenizationResult) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*TokenizationResult) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{1}
}
func (x *TokenizationResult) GetTokenIds() []uint32 {
if x != nil {
return x.TokenIds
}
return nil
}
func (x *TokenizationResult) GetMultimodalFeatures() []*MultiModalFeature {
if x != nil {
return x.MultimodalFeatures
}
return nil
}
type InitializeRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"`
TokenizerName *string `protobuf:"bytes,3,opt,name=tokenizer_name,json=tokenizerName,proto3,oneof" json:"tokenizer_name,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *InitializeRequest) Reset() {
*x = InitializeRequest{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *InitializeRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*InitializeRequest) ProtoMessage() {}
func (x *InitializeRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[2]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*InitializeRequest) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{2}
}
func (x *InitializeRequest) GetModel() string {
if x != nil {
return x.Model
}
return ""
}
func (x *InitializeRequest) GetTokenizerName() string {
if x != nil && x.TokenizerName != nil {
return *x.TokenizerName
}
return ""
}
type InitializeResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"`
ResolvedProvider TokenizerProvider `protobuf:"varint,2,opt,name=resolved_provider,json=resolvedProvider,proto3,enum=hermes.tokenizer.v1.TokenizerProvider" json:"resolved_provider,omitempty"`
ResolvedTokenizerName string `protobuf:"bytes,3,opt,name=resolved_tokenizer_name,json=resolvedTokenizerName,proto3" json:"resolved_tokenizer_name,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *InitializeResponse) Reset() {
*x = InitializeResponse{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *InitializeResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*InitializeResponse) ProtoMessage() {}
func (x *InitializeResponse) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[3]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*InitializeResponse) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{3}
}
func (x *InitializeResponse) GetModel() string {
if x != nil {
return x.Model
}
return ""
}
func (x *InitializeResponse) GetResolvedProvider() TokenizerProvider {
if x != nil {
return x.ResolvedProvider
}
return TokenizerProvider_TOKENIZER_PROVIDER_UNSPECIFIED
}
func (x *InitializeResponse) GetResolvedTokenizerName() string {
if x != nil {
return x.ResolvedTokenizerName
}
return ""
}
type TokenizeRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"`
Prompt string `protobuf:"bytes,2,opt,name=prompt,proto3" json:"prompt,omitempty"`
AddSpecialTokens bool `protobuf:"varint,3,opt,name=add_special_tokens,json=addSpecialTokens,proto3" json:"add_special_tokens,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *TokenizeRequest) Reset() {
*x = TokenizeRequest{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *TokenizeRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*TokenizeRequest) ProtoMessage() {}
func (x *TokenizeRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[4]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*TokenizeRequest) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{4}
}
func (x *TokenizeRequest) GetModel() string {
if x != nil {
return x.Model
}
return ""
}
func (x *TokenizeRequest) GetPrompt() string {
if x != nil {
return x.Prompt
}
return ""
}
func (x *TokenizeRequest) GetAddSpecialTokens() bool {
if x != nil {
return x.AddSpecialTokens
}
return false
}
type TokenizeResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Result *TokenizationResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *TokenizeResponse) Reset() {
*x = TokenizeResponse{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *TokenizeResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*TokenizeResponse) ProtoMessage() {}
func (x *TokenizeResponse) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[5]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*TokenizeResponse) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{5}
}
func (x *TokenizeResponse) GetResult() *TokenizationResult {
if x != nil {
return x.Result
}
return nil
}
type TokenIdList struct {
state protoimpl.MessageState `protogen:"open.v1"`
Values []uint32 `protobuf:"varint,1,rep,packed,name=values,proto3" json:"values,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *TokenIdList) Reset() {
*x = TokenIdList{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *TokenIdList) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*TokenIdList) ProtoMessage() {}
func (x *TokenIdList) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[6]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*TokenIdList) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{6}
}
func (x *TokenIdList) GetValues() []uint32 {
if x != nil {
return x.Values
}
return nil
}
type RenderCompletionRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"`
PromptSource isRenderCompletionRequest_PromptSource `protobuf_oneof:"prompt_source"`
AddSpecialTokens bool `protobuf:"varint,4,opt,name=add_special_tokens,json=addSpecialTokens,proto3" json:"add_special_tokens,omitempty"`
TruncatePromptTokens *int32 `protobuf:"varint,5,opt,name=truncate_prompt_tokens,json=truncatePromptTokens,proto3,oneof" json:"truncate_prompt_tokens,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *RenderCompletionRequest) Reset() {
*x = RenderCompletionRequest{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *RenderCompletionRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*RenderCompletionRequest) ProtoMessage() {}
func (x *RenderCompletionRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[7]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*RenderCompletionRequest) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{7}
}
func (x *RenderCompletionRequest) GetModel() string {
if x != nil {
return x.Model
}
return ""
}
func (x *RenderCompletionRequest) GetPromptSource() isRenderCompletionRequest_PromptSource {
if x != nil {
return x.PromptSource
}
return nil
}
func (x *RenderCompletionRequest) GetPromptText() string {
if x != nil {
if x, ok := x.PromptSource.(*RenderCompletionRequest_PromptText); ok {
return x.PromptText
}
}
return ""
}
func (x *RenderCompletionRequest) GetPromptTokenIds() *TokenIdList {
if x != nil {
if x, ok := x.PromptSource.(*RenderCompletionRequest_PromptTokenIds); ok {
return x.PromptTokenIds
}
}
return nil
}
func (x *RenderCompletionRequest) GetAddSpecialTokens() bool {
if x != nil {
return x.AddSpecialTokens
}
return false
}
func (x *RenderCompletionRequest) GetTruncatePromptTokens() int32 {
if x != nil && x.TruncatePromptTokens != nil {
return *x.TruncatePromptTokens
}
return 0
}
type isRenderCompletionRequest_PromptSource interface {
isRenderCompletionRequest_PromptSource()
}
type RenderCompletionRequest_PromptText struct {
PromptText string `protobuf:"bytes,2,opt,name=prompt_text,json=promptText,proto3,oneof"`
}
type RenderCompletionRequest_PromptTokenIds struct {
PromptTokenIds *TokenIdList `protobuf:"bytes,3,opt,name=prompt_token_ids,json=promptTokenIds,proto3,oneof"`
}
func (*RenderCompletionRequest_PromptText) isRenderCompletionRequest_PromptSource() {}
func (*RenderCompletionRequest_PromptTokenIds) isRenderCompletionRequest_PromptSource() {}
type RenderCompletionResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Result *TokenizationResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *RenderCompletionResponse) Reset() {
*x = RenderCompletionResponse{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *RenderCompletionResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*RenderCompletionResponse) ProtoMessage() {}
func (x *RenderCompletionResponse) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*RenderCompletionResponse) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{8}
}
func (x *RenderCompletionResponse) GetResult() *TokenizationResult {
if x != nil {
return x.Result
}
return nil
}
type MediaRef struct {
state protoimpl.MessageState `protogen:"open.v1"`
Modality Modality `protobuf:"varint,1,opt,name=modality,proto3,enum=hermes.tokenizer.v1.Modality" json:"modality,omitempty"`
Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"`
InlineData []byte `protobuf:"bytes,3,opt,name=inline_data,json=inlineData,proto3" json:"inline_data,omitempty"`
MimeType string `protobuf:"bytes,4,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"`
DetailJson string `protobuf:"bytes,5,opt,name=detail_json,json=detailJson,proto3" json:"detail_json,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *MediaRef) Reset() {
*x = MediaRef{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *MediaRef) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MediaRef) ProtoMessage() {}
func (x *MediaRef) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*MediaRef) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{9}
}
func (x *MediaRef) GetModality() Modality {
if x != nil {
return x.Modality
}
return Modality_MODALITY_UNSPECIFIED
}
func (x *MediaRef) GetUrl() string {
if x != nil {
return x.Url
}
return ""
}
func (x *MediaRef) GetInlineData() []byte {
if x != nil {
return x.InlineData
}
return nil
}
func (x *MediaRef) GetMimeType() string {
if x != nil {
return x.MimeType
}
return ""
}
func (x *MediaRef) GetDetailJson() string {
if x != nil {
return x.DetailJson
}
return ""
}
type ChatContentPart struct {
state protoimpl.MessageState `protogen:"open.v1"`
Part isChatContentPart_Part `protobuf_oneof:"part"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ChatContentPart) Reset() {
*x = ChatContentPart{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ChatContentPart) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ChatContentPart) ProtoMessage() {}
func (x *ChatContentPart) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*ChatContentPart) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{10}
}
func (x *ChatContentPart) GetPart() isChatContentPart_Part {
if x != nil {
return x.Part
}
return nil
}
func (x *ChatContentPart) GetText() string {
if x != nil {
if x, ok := x.Part.(*ChatContentPart_Text); ok {
return x.Text
}
}
return ""
}
func (x *ChatContentPart) GetMedia() *MediaRef {
if x != nil {
if x, ok := x.Part.(*ChatContentPart_Media); ok {
return x.Media
}
}
return nil
}
type isChatContentPart_Part interface {
isChatContentPart_Part()
}
type ChatContentPart_Text struct {
Text string `protobuf:"bytes,1,opt,name=text,proto3,oneof"`
}
type ChatContentPart_Media struct {
Media *MediaRef `protobuf:"bytes,2,opt,name=media,proto3,oneof"`
}
func (*ChatContentPart_Text) isChatContentPart_Part() {}
func (*ChatContentPart_Media) isChatContentPart_Part() {}
type ChatContentParts struct {
state protoimpl.MessageState `protogen:"open.v1"`
Values []*ChatContentPart `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ChatContentParts) Reset() {
*x = ChatContentParts{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ChatContentParts) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ChatContentParts) ProtoMessage() {}
func (x *ChatContentParts) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*ChatContentParts) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{11}
}
func (x *ChatContentParts) GetValues() []*ChatContentPart {
if x != nil {
return x.Values
}
return nil
}
type ChatMessage struct {
state protoimpl.MessageState `protogen:"open.v1"`
Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"`
ContentForm isChatMessage_ContentForm `protobuf_oneof:"content_form"`
Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"`
ToolCallsJson string `protobuf:"bytes,5,opt,name=tool_calls_json,json=toolCallsJson,proto3" json:"tool_calls_json,omitempty"`
ToolCallId string `protobuf:"bytes,6,opt,name=tool_call_id,json=toolCallId,proto3" json:"tool_call_id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ChatMessage) Reset() {
*x = ChatMessage{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ChatMessage) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ChatMessage) ProtoMessage() {}
func (x *ChatMessage) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*ChatMessage) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{12}
}
func (x *ChatMessage) GetRole() string {
if x != nil {
return x.Role
}
return ""
}
func (x *ChatMessage) GetContentForm() isChatMessage_ContentForm {
if x != nil {
return x.ContentForm
}
return nil
}
func (x *ChatMessage) GetContent() string {
if x != nil {
if x, ok := x.ContentForm.(*ChatMessage_Content); ok {
return x.Content
}
}
return ""
}
func (x *ChatMessage) GetContentParts() *ChatContentParts {
if x != nil {
if x, ok := x.ContentForm.(*ChatMessage_ContentParts); ok {
return x.ContentParts
}
}
return nil
}
func (x *ChatMessage) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *ChatMessage) GetToolCallsJson() string {
if x != nil {
return x.ToolCallsJson
}
return ""
}
func (x *ChatMessage) GetToolCallId() string {
if x != nil {
return x.ToolCallId
}
return ""
}
type isChatMessage_ContentForm interface {
isChatMessage_ContentForm()
}
type ChatMessage_Content struct {
Content string `protobuf:"bytes,2,opt,name=content,proto3,oneof"`
}
type ChatMessage_ContentParts struct {
ContentParts *ChatContentParts `protobuf:"bytes,3,opt,name=content_parts,json=contentParts,proto3,oneof"`
}
func (*ChatMessage_Content) isChatMessage_ContentForm() {}
func (*ChatMessage_ContentParts) isChatMessage_ContentForm() {}
type RenderChatCompletionRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"`
Messages []*ChatMessage `protobuf:"bytes,2,rep,name=messages,proto3" json:"messages,omitempty"`
ChatTemplate string `protobuf:"bytes,3,opt,name=chat_template,json=chatTemplate,proto3" json:"chat_template,omitempty"`
ChatTemplateKwargsJson string `protobuf:"bytes,4,opt,name=chat_template_kwargs_json,json=chatTemplateKwargsJson,proto3" json:"chat_template_kwargs_json,omitempty"`
ToolsJson string `protobuf:"bytes,5,opt,name=tools_json,json=toolsJson,proto3" json:"tools_json,omitempty"`
ToolChoiceJson string `protobuf:"bytes,6,opt,name=tool_choice_json,json=toolChoiceJson,proto3" json:"tool_choice_json,omitempty"`
AddGenerationPrompt bool `protobuf:"varint,7,opt,name=add_generation_prompt,json=addGenerationPrompt,proto3" json:"add_generation_prompt,omitempty"`
ContinueFinalMessage bool `protobuf:"varint,8,opt,name=continue_final_message,json=continueFinalMessage,proto3" json:"continue_final_message,omitempty"`
MmProcessorKwargsJson string `protobuf:"bytes,9,opt,name=mm_processor_kwargs_json,json=mmProcessorKwargsJson,proto3" json:"mm_processor_kwargs_json,omitempty"`
MediaIoKwargsJson string `protobuf:"bytes,10,opt,name=media_io_kwargs_json,json=mediaIoKwargsJson,proto3" json:"media_io_kwargs_json,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *RenderChatCompletionRequest) Reset() {
*x = RenderChatCompletionRequest{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *RenderChatCompletionRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*RenderChatCompletionRequest) ProtoMessage() {}
func (x *RenderChatCompletionRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*RenderChatCompletionRequest) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{13}
}
func (x *RenderChatCompletionRequest) GetModel() string {
if x != nil {
return x.Model
}
return ""
}
func (x *RenderChatCompletionRequest) GetMessages() []*ChatMessage {
if x != nil {
return x.Messages
}
return nil
}
func (x *RenderChatCompletionRequest) GetChatTemplate() string {
if x != nil {
return x.ChatTemplate
}
return ""
}
func (x *RenderChatCompletionRequest) GetChatTemplateKwargsJson() string {
if x != nil {
return x.ChatTemplateKwargsJson
}
return ""
}
func (x *RenderChatCompletionRequest) GetToolsJson() string {
if x != nil {
return x.ToolsJson
}
return ""
}
func (x *RenderChatCompletionRequest) GetToolChoiceJson() string {
if x != nil {
return x.ToolChoiceJson
}
return ""
}
func (x *RenderChatCompletionRequest) GetAddGenerationPrompt() bool {
if x != nil {
return x.AddGenerationPrompt
}
return false
}
func (x *RenderChatCompletionRequest) GetContinueFinalMessage() bool {
if x != nil {
return x.ContinueFinalMessage
}
return false
}
func (x *RenderChatCompletionRequest) GetMmProcessorKwargsJson() string {
if x != nil {
return x.MmProcessorKwargsJson
}
return ""
}
func (x *RenderChatCompletionRequest) GetMediaIoKwargsJson() string {
if x != nil {
return x.MediaIoKwargsJson
}
return ""
}
type RenderChatCompletionResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Result *TokenizationResult `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *RenderChatCompletionResponse) Reset() {
*x = RenderChatCompletionResponse{}
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *RenderChatCompletionResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*RenderChatCompletionResponse) ProtoMessage() {}
func (x *RenderChatCompletionResponse) ProtoReflect() protoreflect.Message {
mi := &file_api_tokenizer_v1_tokenizer_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
func (*RenderChatCompletionResponse) Descriptor() ([]byte, []int) {
return file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP(), []int{14}
}
func (x *RenderChatCompletionResponse) GetResult() *TokenizationResult {
if x != nil {
return x.Result
}
return nil
}
var File_api_tokenizer_v1_tokenizer_proto protoreflect.FileDescriptor
const file_api_tokenizer_v1_tokenizer_proto_rawDesc = "" +
"\n" +
" api/tokenizer/v1/tokenizer.proto\x12\x13hermes.tokenizer.v1\"\x92\x01\n" +
"\x11MultiModalFeature\x129\n" +
"\bmodality\x18\x01 \x01(\x0e2\x1d.hermes.tokenizer.v1.ModalityR\bmodality\x12\x12\n" +
"\x04hash\x18\x02 \x01(\tR\x04hash\x12\x16\n" +
"\x06offset\x18\x03 \x01(\x05R\x06offset\x12\x16\n" +
"\x06length\x18\x04 \x01(\x05R\x06length\"\x8a\x01\n" +
"\x12TokenizationResult\x12\x1b\n" +
"\ttoken_ids\x18\x01 \x03(\rR\btokenIds\x12W\n" +
"\x13multimodal_features\x18\x02 \x03(\v2&.hermes.tokenizer.v1.MultiModalFeatureR\x12multimodalFeatures\"h\n" +
"\x11InitializeRequest\x12\x14\n" +
"\x05model\x18\x01 \x01(\tR\x05model\x12*\n" +
"\x0etokenizer_name\x18\x03 \x01(\tH\x00R\rtokenizerName\x88\x01\x01B\x11\n" +
"\x0f_tokenizer_name\"\xb7\x01\n" +
"\x12InitializeResponse\x12\x14\n" +
"\x05model\x18\x01 \x01(\tR\x05model\x12S\n" +
"\x11resolved_provider\x18\x02 \x01(\x0e2&.hermes.tokenizer.v1.TokenizerProviderR\x10resolvedProvider\x126\n" +
"\x17resolved_tokenizer_name\x18\x03 \x01(\tR\x15resolvedTokenizerName\"m\n" +
"\x0fTokenizeRequest\x12\x14\n" +
"\x05model\x18\x01 \x01(\tR\x05model\x12\x16\n" +
"\x06prompt\x18\x02 \x01(\tR\x06prompt\x12,\n" +
"\x12add_special_tokens\x18\x03 \x01(\bR\x10addSpecialTokens\"S\n" +
"\x10TokenizeResponse\x12?\n" +
"\x06result\x18\x01 \x01(\v2'.hermes.tokenizer.v1.TokenizationResultR\x06result\"%\n" +
"\vTokenIdList\x12\x16\n" +
"\x06values\x18\x01 \x03(\rR\x06values\"\xb5\x02\n" +
"\x17RenderCompletionRequest\x12\x14\n" +
"\x05model\x18\x01 \x01(\tR\x05model\x12!\n" +
"\vprompt_text\x18\x02 \x01(\tH\x00R\n" +
"promptText\x12L\n" +
"\x10prompt_token_ids\x18\x03 \x01(\v2 .hermes.tokenizer.v1.TokenIdListH\x00R\x0epromptTokenIds\x12,\n" +
"\x12add_special_tokens\x18\x04 \x01(\bR\x10addSpecialTokens\x129\n" +
"\x16truncate_prompt_tokens\x18\x05 \x01(\x05H\x01R\x14truncatePromptTokens\x88\x01\x01B\x0f\n" +
"\rprompt_sourceB\x19\n" +
"\x17_truncate_prompt_tokens\"[\n" +
"\x18RenderCompletionResponse\x12?\n" +
"\x06result\x18\x01 \x01(\v2'.hermes.tokenizer.v1.TokenizationResultR\x06result\"\xb6\x01\n" +
"\bMediaRef\x129\n" +
"\bmodality\x18\x01 \x01(\x0e2\x1d.hermes.tokenizer.v1.ModalityR\bmodality\x12\x10\n" +
"\x03url\x18\x02 \x01(\tR\x03url\x12\x1f\n" +
"\vinline_data\x18\x03 \x01(\fR\n" +
"inlineData\x12\x1b\n" +
"\tmime_type\x18\x04 \x01(\tR\bmimeType\x12\x1f\n" +
"\vdetail_json\x18\x05 \x01(\tR\n" +
"detailJson\"f\n" +
"\x0fChatContentPart\x12\x14\n" +
"\x04text\x18\x01 \x01(\tH\x00R\x04text\x125\n" +
"\x05media\x18\x02 \x01(\v2\x1d.hermes.tokenizer.v1.MediaRefH\x00R\x05mediaB\x06\n" +
"\x04part\"P\n" +
"\x10ChatContentParts\x12<\n" +
"\x06values\x18\x01 \x03(\v2$.hermes.tokenizer.v1.ChatContentPartR\x06values\"\xf9\x01\n" +
"\vChatMessage\x12\x12\n" +
"\x04role\x18\x01 \x01(\tR\x04role\x12\x1a\n" +
"\acontent\x18\x02 \x01(\tH\x00R\acontent\x12L\n" +
"\rcontent_parts\x18\x03 \x01(\v2%.hermes.tokenizer.v1.ChatContentPartsH\x00R\fcontentParts\x12\x12\n" +
"\x04name\x18\x04 \x01(\tR\x04name\x12&\n" +
"\x0ftool_calls_json\x18\x05 \x01(\tR\rtoolCallsJson\x12 \n" +
"\ftool_call_id\x18\x06 \x01(\tR\n" +
"toolCallIdB\x0e\n" +
"\fcontent_form\"\xee\x03\n" +
"\x1bRenderChatCompletionRequest\x12\x14\n" +
"\x05model\x18\x01 \x01(\tR\x05model\x12<\n" +
"\bmessages\x18\x02 \x03(\v2 .hermes.tokenizer.v1.ChatMessageR\bmessages\x12#\n" +
"\rchat_template\x18\x03 \x01(\tR\fchatTemplate\x129\n" +
"\x19chat_template_kwargs_json\x18\x04 \x01(\tR\x16chatTemplateKwargsJson\x12\x1d\n" +
"\n" +
"tools_json\x18\x05 \x01(\tR\ttoolsJson\x12(\n" +
"\x10tool_choice_json\x18\x06 \x01(\tR\x0etoolChoiceJson\x122\n" +
"\x15add_generation_prompt\x18\a \x01(\bR\x13addGenerationPrompt\x124\n" +
"\x16continue_final_message\x18\b \x01(\bR\x14continueFinalMessage\x127\n" +
"\x18mm_processor_kwargs_json\x18\t \x01(\tR\x15mmProcessorKwargsJson\x12/\n" +
"\x14media_io_kwargs_json\x18\n" +
" \x01(\tR\x11mediaIoKwargsJson\"_\n" +
"\x1cRenderChatCompletionResponse\x12?\n" +
"\x06result\x18\x01 \x01(\v2'.hermes.tokenizer.v1.TokenizationResultR\x06result*\x9b\x01\n" +
"\x11TokenizerProvider\x12\"\n" +
"\x1eTOKENIZER_PROVIDER_UNSPECIFIED\x10\x00\x12\x1b\n" +
"\x17TOKENIZER_PROVIDER_AUTO\x10\x01\x12\"\n" +
"\x1eTOKENIZER_PROVIDER_HUGGINGFACE\x10\x02\x12!\n" +
"\x1dTOKENIZER_PROVIDER_MODELSCOPE\x10\x03*8\n" +
"\bModality\x12\x18\n" +
"\x14MODALITY_UNSPECIFIED\x10\x00\x12\x12\n" +
"\x0eMODALITY_IMAGE\x10\x012\xbb\x03\n" +
"\x13TokenizationService\x12]\n" +
"\n" +
"Initialize\x12&.hermes.tokenizer.v1.InitializeRequest\x1a'.hermes.tokenizer.v1.InitializeResponse\x12W\n" +
"\bTokenize\x12$.hermes.tokenizer.v1.TokenizeRequest\x1a%.hermes.tokenizer.v1.TokenizeResponse\x12o\n" +
"\x10RenderCompletion\x12,.hermes.tokenizer.v1.RenderCompletionRequest\x1a-.hermes.tokenizer.v1.RenderCompletionResponse\x12{\n" +
"\x14RenderChatCompletion\x120.hermes.tokenizer.v1.RenderChatCompletionRequest\x1a1.hermes.tokenizer.v1.RenderChatCompletionResponseB,Z*hermes-router/api/tokenizer/v1;tokenizerv1b\x06proto3"
var (
file_api_tokenizer_v1_tokenizer_proto_rawDescOnce sync.Once
file_api_tokenizer_v1_tokenizer_proto_rawDescData []byte
)
func file_api_tokenizer_v1_tokenizer_proto_rawDescGZIP() []byte {
file_api_tokenizer_v1_tokenizer_proto_rawDescOnce.Do(func() {
file_api_tokenizer_v1_tokenizer_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_api_tokenizer_v1_tokenizer_proto_rawDesc), len(file_api_tokenizer_v1_tokenizer_proto_rawDesc)))
})
return file_api_tokenizer_v1_tokenizer_proto_rawDescData
}
var file_api_tokenizer_v1_tokenizer_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_api_tokenizer_v1_tokenizer_proto_msgTypes = make([]protoimpl.MessageInfo, 15)
var file_api_tokenizer_v1_tokenizer_proto_goTypes = []any{
(TokenizerProvider)(0),
(Modality)(0),
(*MultiModalFeature)(nil),
(*TokenizationResult)(nil),
(*InitializeRequest)(nil),
(*InitializeResponse)(nil),
(*TokenizeRequest)(nil),
(*TokenizeResponse)(nil),
(*TokenIdList)(nil),
(*RenderCompletionRequest)(nil),
(*RenderCompletionResponse)(nil),
(*MediaRef)(nil),
(*ChatContentPart)(nil),
(*ChatContentParts)(nil),
(*ChatMessage)(nil),
(*RenderChatCompletionRequest)(nil),
(*RenderChatCompletionResponse)(nil),
}
var file_api_tokenizer_v1_tokenizer_proto_depIdxs = []int32{
1,
2,
0,
3,
8,
3,
1,
11,
12,
13,
14,
3,
4,
6,
9,
15,
5,
7,
10,
16,
16,
12,
12,
12,
0,
}
func init() { file_api_tokenizer_v1_tokenizer_proto_init() }
func file_api_tokenizer_v1_tokenizer_proto_init() {
if File_api_tokenizer_v1_tokenizer_proto != nil {
return
}
file_api_tokenizer_v1_tokenizer_proto_msgTypes[2].OneofWrappers = []any{}
file_api_tokenizer_v1_tokenizer_proto_msgTypes[7].OneofWrappers = []any{
(*RenderCompletionRequest_PromptText)(nil),
(*RenderCompletionRequest_PromptTokenIds)(nil),
}
file_api_tokenizer_v1_tokenizer_proto_msgTypes[10].OneofWrappers = []any{
(*ChatContentPart_Text)(nil),
(*ChatContentPart_Media)(nil),
}
file_api_tokenizer_v1_tokenizer_proto_msgTypes[12].OneofWrappers = []any{
(*ChatMessage_Content)(nil),
(*ChatMessage_ContentParts)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_tokenizer_v1_tokenizer_proto_rawDesc), len(file_api_tokenizer_v1_tokenizer_proto_rawDesc)),
NumEnums: 2,
NumMessages: 15,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_api_tokenizer_v1_tokenizer_proto_goTypes,
DependencyIndexes: file_api_tokenizer_v1_tokenizer_proto_depIdxs,
EnumInfos: file_api_tokenizer_v1_tokenizer_proto_enumTypes,
MessageInfos: file_api_tokenizer_v1_tokenizer_proto_msgTypes,
}.Build()
File_api_tokenizer_v1_tokenizer_proto = out.File
file_api_tokenizer_v1_tokenizer_proto_goTypes = nil
file_api_tokenizer_v1_tokenizer_proto_depIdxs = nil
}