AnalysisEngine

[fundon]

CPU 100% and too more memory

[jdm]

Can you be more specific about this? Is there a webpage that reproduces this issue in Servo, or Rust code that demonstrates the problem?

[fundon]

1. Web

https://github.com/kelvinau/circular-audio-wave

The first example is ok, it can be played. But second example is bad. On chrome all ok.

2. Rust

I tested the https://github.com/servo/media/blob/master/examples/audio_decoder.rs.
And add some codes to it.

extern crate servo_media;
extern crate servo_media_auto;

use servo_media::audio::analyser_node::AnalysisEngine;
use servo_media::audio::buffer_source_node::{AudioBuffer, AudioBufferSourceNodeMessage};
use servo_media::audio::context::{AudioContextOptions, RealTimeAudioContextOptions};
use servo_media::audio::decoder::AudioDecoderCallbacks;
use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage};
use servo_media::{ClientContextId, ServoMedia};
use std::env;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::sync::mpsc;
use std::sync::{Arc, Mutex};
use std::{thread, time};

fn run_example(servo_media: Arc<ServoMedia>) {
    let options = <RealTimeAudioContextOptions>::default();
    let sample_rate = options.sample_rate;
    let context = servo_media.create_audio_context(
        &ClientContextId::build(1, 1),
        AudioContextOptions::RealTimeAudioContext(options),
    );
    let context = context.lock().unwrap();
    let args: Vec<_> = env::args().collect();
    let default = "./examples/resources/viper_cut.ogg";
    let filename: &str = if args.len() == 2 {
        args[1].as_ref()
    } else if Path::new(default).exists() {
        default
    } else {
        panic!("Usage: cargo run --bin audio_decoder <file_path>")
    };
    let mut file = File::open(filename).unwrap();
    let mut bytes = vec![];
    file.read_to_end(&mut bytes).unwrap();
    let decoded_audio: Arc<Mutex<Vec<Vec<f32>>>> = Arc::new(Mutex::new(Vec::new()));
    let decoded_audio_ = decoded_audio.clone();
    let decoded_audio__ = decoded_audio.clone();
    let (sender, receiver) = mpsc::channel();
    let callbacks = AudioDecoderCallbacks::new()
        .eos(move || {
            sender.send(()).unwrap();
        })
        .error(|e| {
            eprintln!("Error decoding audio {:?}", e);
        })
        .progress(move |buffer, channel| {
            let mut decoded_audio = decoded_audio_.lock().unwrap();
            decoded_audio[(channel - 1) as usize].extend_from_slice((*buffer).as_ref());
        })
        .ready(move |channels| {
            println!("There are {:?} audio channels", channels);
            decoded_audio__
                .lock()
                .unwrap()
                .resize(channels as usize, Vec::new());
        })
        .build();
    context.decode_audio_data(bytes.to_vec(), callbacks);
    println!("Decoding audio");
    receiver.recv().unwrap();
    println!("Audio decoded");
    let buffer_source = context.create_node(
        AudioNodeInit::AudioBufferSourceNode(Default::default()),
        Default::default(),
    );
    let dest = context.dest_node();
    let analyser = Arc::new(Mutex::new(AnalysisEngine::new(512, 0.6, -30.0, -100.0)));
    let (sender, receiver) = mpsc::channel();
    let analyser_node = context.create_node(
        AudioNodeInit::AnalyserNode(Box::new(move |block| {
            let _ = sender.send(dbg!(block));
        })),
        Default::default(),
    );
    let analyser = analyser.clone();
    thread::spawn(move || {
        loop {
            if let Ok(block) = receiver.recv() {
                let mut a = analyser.lock().unwrap();
                a.push(block);
                let mut data = [0u8; 512];
                a.fill_byte_frequency_data(&mut data);
                // dbg!(data);
            }
        }
    });
    context.connect_ports(buffer_source.output(0), analyser_node.input(0));
    context.connect_ports(analyser_node.output(0), dest.input(0));
    context.message_node(
        buffer_source,
        AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)),
    );
    context.message_node(
        buffer_source,
        AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer(Some(
            AudioBuffer::from_buffers(decoded_audio.lock().unwrap().to_vec(), sample_rate),
        ))),
    );
    let _ = context.resume();
    thread::sleep(time::Duration::from_millis(3000));
    let _ = context.suspend();
    thread::sleep(time::Duration::from_millis(3000));
    let _ = context.resume();
    thread::sleep(time::Duration::from_millis(60000));
    let _ = context.close();
}

fn main() {
    ServoMedia::init::<servo_media_auto::Backend>();
    if let Ok(servo_media) = ServoMedia::get() {
        run_example(servo_media);
    } else {
        unreachable!()
    }
}

[fundon]

I think that the FFT need refactor. 🤣

[jdm]

@fundon Why did you close the issue?

[fundon]

@jdm Sorry, I opened it in the GitHub App and clicked the close button by mistake. And GitHub App unable to reopen. 😂