VVladimir Mandicupdate nunchaku-lite
05eef5e9创建于 8月19日历史提交
import os
import diffusers
import transformers
from modules import shared, devices, sd_models, model_quant, sd_hijack_te
from modules.logger import log
from pipelines import generic


def load_flux(checkpoint_info, diffusers_load_config=None):
    if diffusers_load_config is None:
        diffusers_load_config = {}
    repo_id = sd_models.path_to_repo(checkpoint_info)
    sd_models.hf_auth_check(checkpoint_info, force=True)

    if 'Fill' in repo_id:
        cls_name = diffusers.FluxFillPipeline
    elif 'Canny' in repo_id:
        cls_name = diffusers.FluxControlPipeline
    elif 'Depth' in repo_id:
        cls_name = diffusers.FluxControlPipeline
    elif 'Kontext' in repo_id:
        cls_name = diffusers.FluxKontextPipeline
        diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["flux1kontext"] = diffusers.FluxKontextPipeline
        diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["flux1kontext"] = diffusers.FluxKontextPipeline
        diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["flux1kontext"] = diffusers.FluxKontextInpaintPipeline
    else:
        cls_name = diffusers.FluxPipeline

    from pipelines.flux import flux_lora
    flux_lora.apply_patch()

    load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
    log.debug(f'Load model: type=Flux repo="{repo_id}" cls={cls_name.__name__} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')

    # optional teacache patch
    if shared.opts.teacache_enabled and not model_quant.check_nunchaku('Model'):
        from modules import teacache
        log.debug(f'Transformers cache: type=teacache patch=forward cls={diffusers.FluxTransformer2DModel.__name__}')
        diffusers.FluxTransformer2DModel.forward = teacache.teacache_flux_forward # patch must be done before transformer is loaded

    transformer = None
    text_encoder_2 = None

    # handle transformer svdquant if available, t5 is handled inside load_text_encoder
    if transformer is None and model_quant.check_nunchaku('Model'):
        from pipelines.flux.flux_nunchaku import load_flux_nunchaku
        transformer = load_flux_nunchaku(repo_id)

    # finally load transformer and text encoder if not already loaded
    if transformer is None:
        transformer = generic.load_transformer(repo_id, cls_name=diffusers.FluxTransformer2DModel, load_config=diffusers_load_config)
    if text_encoder_2 is None:
        text_encoder_2 = generic.load_text_encoder(repo_id, cls_name=transformers.T5EncoderModel, load_config=diffusers_load_config)

    if repo_id is None or repo_id.lower() == 'none':
        return None
    pipe = cls_name.from_pretrained(
        repo_id,
        transformer=transformer,
        text_encoder_2=text_encoder_2,
        cache_dir=shared.opts.diffusers_dir,
        **load_args,
    )

    generic.load_vae_override(pipe, diffusers_load_config)

    if os.environ.get('SD_REMOTE_T5', None) is not None:
        from modules import sd_te_remote
        log.warning('Remote-TE: applying patch')
        pipe._get_t5_prompt_embeds = sd_te_remote.get_t5_prompt_embeds # pylint: disable=protected-access
        pipe.text_encoder_2 = None

    del text_encoder_2
    del transformer

    # optional first-block patch
    if shared.opts.teacache_enabled and model_quant.check_nunchaku('Model'):
        from nunchaku.caching.diffusers_adapters import apply_cache_on_pipe
        apply_cache_on_pipe(pipe, residual_diff_threshold=0.12)

    sd_hijack_te.init_hijack(pipe)

    devices.torch_gc(force=True, reason='load')
    return pipe