training audioldm1 and not audioldm2

[kmaro2345]

i want to train only audioldm1 and not audioldm2 how can i do that !

[haoheliu]

You can achieve that by correctly configure the yaml file, for example, by only using the CLAP encoders as conditioning signals

[kmaro2345]

metadata_root: "./data/dataset/metadata/dataset_root.json"
log_directory: "./log/latent_diffusion"
project: "audioldm"
precision: "high"

variables:
sampling_rate: &sampling_rate 16000
mel_bins: &mel_bins 64
latent_embed_dim: &latent_embed_dim 8
latent_t_size: &latent_t_size 256 # TODO might need to change
latent_f_size: &latent_f_size 16
in_channels: &unet_in_channels 8
optimize_ddpm_parameter: &optimize_ddpm_parameter true
optimize_gpt: &optimize_gpt true
warmup_steps: &warmup_steps 2000

data:
train: ["audiocaps"]
val: "audiocaps"
test: "audiocaps"
class_label_indices: "audioset_eval_subset"
dataloader_add_ons: []

step:
validation_every_n_epochs: 5
save_checkpoint_every_n_steps: 12505

limit_val_batches: 2

max_steps: 375150
save_top_k: 1

preprocessing:
audio:
sampling_rate: *sampling_rate
max_wav_value: 32768.0
duration: 10.24
stft:
filter_length: 1024
hop_length: 160
win_length: 1024
mel:
n_mel_channels: *mel_bins
mel_fmin: 0
mel_fmax: 8000

augmentation:
mixup: 0.0

model:
target: audioldm_train.modules.latent_diffusion.ddpm.LatentDiffusion
params:
# Autoencoder
first_stage_config:
base_learning_rate: 8.0e-06
target: audioldm_train.modules.latent_encoder.autoencoder.AutoencoderKL
params:
reload_from_ckpt: "data/checkpoints/vae_mel_16k_64bins.ckpt"
sampling_rate: *sampling_rate
batchsize: 4
monitor: val/rec_loss
image_key: fbank
subband: 1
embed_dim: *latent_embed_dim
time_shuffle: 1
lossconfig:
target: audioldm_train.losses.LPIPSWithDiscriminator
params:
disc_start: 50001
kl_weight: 1000.0
disc_weight: 0.5
disc_in_channels: 1
ddconfig:
double_z: true
mel_bins: *mel_bins # The frequency bins of mel spectrogram
z_channels: 8
resolution: 256
downsample_time: false
in_channels: 1
out_ch: 1
ch: 128
ch_mult:
- 1
- 2
- 4
num_res_blocks: 2
attn_resolutions: []
dropout: 0.0

# Other parameters
base_learning_rate: 1.0e-4
warmup_steps: *warmup_steps
optimize_ddpm_parameter: *optimize_ddpm_parameter
sampling_rate: *sampling_rate
batchsize: 2
linear_start: 0.0015
linear_end: 0.0195
num_timesteps_cond: 1
log_every_t: 200
timesteps: 1000
unconditional_prob_cfg: 0.1
parameterization: eps # [eps, x0, v]
first_stage_key: fbank
latent_t_size: *latent_t_size # TODO might need to change
latent_f_size: *latent_f_size
channels: *latent_embed_dim # TODO might need to change
monitor: val/loss_simple_ema
scale_by_std: true
unet_config:
  target: audioldm_train.modules.diffusionmodules.openaimodel.UNetModel
  params:
    image_size: 64 
    extra_film_condition_dim: 512 # If you use film as extra condition, set this parameter. For example if you have two conditioning vectors each have dimension 512, then this number would be 1024
    # context_dim: 
    # - 768
    in_channels: *unet_in_channels # The input channel of the UNet model
    out_channels: *latent_embed_dim # TODO might need to change
    model_channels: 128 # TODO might need to change
    attention_resolutions:
    - 8
    - 4
    - 2
    num_res_blocks: 2
    channel_mult: 
    - 1
    - 2
    - 3
    - 5
    num_head_channels: 32
    use_spatial_transformer: true
    transformer_depth: 1
    extra_sa_layer: false

cond_stage_config:
  film_clap_cond1:
    cond_stage_key: text
    conditioning_key: film
    target: audioldm_train.conditional_models.CLAPAudioEmbeddingClassifierFreev2
    params:
      pretrained_path: data/checkpoints/clap_htsat_tiny.pt
      sampling_rate: 16000
      embed_mode: text # or text
      amodel: HTSAT-tiny

evaluation_params:
  unconditional_guidance_scale: 3.5
  ddim_sampling_steps: 200
  n_candidates_per_samples: 3

is this config file use the clap encoders as conditioning signals?