0e35079f创建于 6月29日历史提交
import os
import torch
import transformers
import diffusers
from huggingface_hub import hf_hub_download
from modules import shared, devices, sd_models, model_quant, sd_hijack_te
from modules.logger import log
from pipelines import generic


def init_nunchaku():
    import nunchaku
    if not hasattr(nunchaku, 'NunchakuZImageTransformer2DModel'): # not present in older versions of nunchaku
        return None
    nunchaku_precision = nunchaku.utils.get_precision()
    nunchaku_rank = 128
    nunchaku_repo = f"nunchaku-ai/nunchaku-z-image-turbo/svdq-{nunchaku_precision}_r{nunchaku_rank}-z-image-turbo.safetensors"
    repo_id, filename = nunchaku_repo.rsplit('/', 1)
    log.debug(f'Load module: quant=Nunchaku module=transformer repo="{nunchaku_repo}" attention={shared.opts.nunchaku_attention}')
    local_path = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=shared.opts.hfcache_dir)
    transformer = nunchaku.NunchakuZImageTransformer2DModel.from_pretrained( # pylint: disable=no-member
        local_path,
        cache_dir=shared.opts.hfcache_dir,
        torch_dtype=devices.dtype,
    )
    return transformer


def load_z_image(checkpoint_info, diffusers_load_config=None):

    if diffusers_load_config is None:
        diffusers_load_config = {}
    repo_id = sd_models.path_to_repo(checkpoint_info)
    sd_models.hf_auth_check(checkpoint_info)

    if devices.dtype == torch.float16:
        from pipelines.z_image.patch_fp16 import apply_patches
        apply_patches()

    load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
    log.debug(f'Load model: type=ZImage repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={diffusers_load_config}')

    transformer = None
    if model_quant.check_nunchaku('Model'): # only available model
        transformer = init_nunchaku()
    if transformer is None:
        transformer = generic.load_transformer(repo_id, cls_name=diffusers.ZImageTransformer2DModel, load_config=diffusers_load_config)

    text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3ForCausalLM, load_config=diffusers_load_config)

    if repo_id is None or repo_id.lower() == 'none':
        return None
    if os.path.exists(repo_id): # local file so map to default repo
        repo_id = generic.transformers_map.get('ZImageTransformer2DModel', repo_id)

    if transformer is None:
        log.error(f'Load model: type=ZImage repo="{repo_id}" failed to load transformer ')
        return None
    if text_encoder is None:
        log.error(f'Load model: type=ZImage repo="{repo_id}" failed to load text encoder ')
        return None

    pipe = diffusers.ZImagePipeline.from_pretrained(
        repo_id,
        cache_dir=shared.opts.diffusers_dir,
        transformer=transformer,
        text_encoder=text_encoder,
        **load_args,
    )

    generic.load_vae_override(pipe, diffusers_load_config)

    del transformer
    del text_encoder
    sd_hijack_te.init_hijack(pipe)

    devices.torch_gc(force=True, reason='load')
    return pipe