Repository navigation
Expand file tree
/
Copy pathwan.cpp
More file actions
119 lines (117 loc) · 7.13 KB
/
Copy pathwan.cpp
File metadata and controls
119 lines (117 loc) · 7.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#include "model/diffusion/wan.hpp"
#include "conditioning/conditioner.hpp"
#include "model/audio/wav2vec2.hpp"
#include "model_definition.h"
#include "pipeline/model_builders.h"
static bool build_wan(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& result) {
const auto* sd_ctx_params = &ctx.params;
const auto& tensor_storage_map = ctx.tensor_storage_map;
const auto version = ctx.version;
const auto& weight_manager = ctx.weight_manager;
result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
tensor_storage_map,
true,
0,
true,
weight_manager);
result.diffusion = std::make_shared<WAN::WanRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.diffusion_model",
version,
weight_manager);
if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) {
result.high_noise_diffusion = std::make_shared<WAN::WanRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
tensor_storage_map,
"model.high_noise_diffusion_model",
version,
weight_manager);
}
if (result.diffusion->get_desc() == "Wan2.1-I2V-14B" ||
result.diffusion->get_desc() == "Wan2.1-FLF2V-14B" ||
result.diffusion->get_desc() == "Wan2.1-I2V-1.3B") {
if (!sd::model_builders::ensure_backend_pair(ctx.backends, SDBackendModule::CLIP_VISION)) {
return false;
}
result.clip_vision = std::make_shared<FrozenCLIPVisionEmbedder>(ctx.backends.runtime_backend(SDBackendModule::CLIP_VISION),
tensor_storage_map,
weight_manager);
}
if (version == VERSION_WAN2_2_S2V &&
tensor_storage_map.count("wav2vec2.encoder.layer_norm.bias") > 0) {
if (!sd::model_builders::ensure_backend_pair(ctx.backends, SDBackendModule::AUDIO_ENCODER)) {
return false;
}
result.audio_encoder = std::make_shared<Wav2Vec2::Wav2Vec2ModelRunner>(ctx.backends.runtime_backend(SDBackendModule::AUDIO_ENCODER),
tensor_storage_map,
"wav2vec2.",
weight_manager);
}
return true;
}
const std::vector<ModelDefinition>& wan_definitions() {
static const std::vector<ModelDefinition> definitions = [] {
std::vector<ModelDefinition> result;
{
ModelDefinition definition;
definition.version = VERSION_WAN2;
definition.geometry = {2, 16, 3};
definition.name = "Wan 2.x";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
definition.sampling = {FLOW_PRED, 5.f};
definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
definition.sampling.default_scheduler = DISCRETE_SCHEDULER;
definition.build = build_wan;
result.push_back(std::move(definition));
}
{
ModelDefinition definition;
definition.version = VERSION_WAN2_2_I2V;
definition.geometry = {2, 16, 3};
definition.name = "Wan 2.2 I2V";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
definition.sampling = {FLOW_PRED, 5.f};
definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
definition.sampling.default_scheduler = DISCRETE_SCHEDULER;
definition.build = build_wan;
result.push_back(std::move(definition));
}
{
ModelDefinition definition;
definition.version = VERSION_WAN2_2_TI2V;
definition.geometry = {2, 48, 3};
definition.name = "Wan 2.2 TI2V";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
definition.sampling = {FLOW_PRED, 5.f};
definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
definition.sampling.default_scheduler = DISCRETE_SCHEDULER;
definition.build = build_wan;
result.push_back(std::move(definition));
}
{
ModelDefinition definition;
definition.version = VERSION_WAN2_2_S2V;
definition.geometry = {2, 16, 3};
definition.name = "Wan 2.2 S2V";
definition.traits.is_dit = true;
definition.traits.supports_video_generation = true;
definition.traits.vae_kind = ModelVAEKind::WAN;
definition.traits.video_tae = true;
definition.sampling = {FLOW_PRED, 5.f};
definition.sampling.default_sample_method = EULER_SAMPLE_METHOD;
definition.sampling.default_scheduler = DISCRETE_SCHEDULER;
definition.build = build_wan;
result.push_back(std::move(definition));
}
return result;
}();
return definitions;
}