{
"dataset_param": {
"dataset_type": "feature",
"basic_parameters": {
"data_path": "./sora_features/data.jsonl",
"data_folder": "./sora_features/"
},
"preprocess_parameters": {
"video_processor_type": "CogVideoXProcessor",
"video_reader_type": "DecordVideo",
"train_fps": 8,
"skip_frame_num": 0,
"num_frames": 136,
"max_height": 544,
"max_width": 992,
"train_pipeline": {
"video": [],
"image": []
}
},
"skip_define_frame_index": true,
"use_text_processer": true,
"enable_text_preprocessing": false,
"tokenizer_config": [
{
"autotokenizer_name": "stepchat",
"hub_backend": "hf",
"from_pretrained": "./weights/step_llm/step1_chat_tokenizer.model",
"model_max_length": 320
},
{
"autotokenizer_name": "BertTokenizer",
"hub_backend": "hf",
"from_pretrained": "./weights/hunyuan_clip/tokenizer",
"model_max_length": 77
}
]
},
"dataloader_param": {
"dataloader_mode": "sampler",
"sampler_type": "SequentialSampler",
"shuffle": false,
"drop_last": true,
"pin_memory": true,
"group_frame": false,
"group_resolution": false,
"collate_param": {}
}
}