error transcribing audio
Closed this issue · 4 comments
this is the terminial log:
Error transcribing audio (attempt 2): Error: HTTP error! status: 500
at transcribeAudio (file:///D:/dev/test/brainrot.js/generate/transcribe.mjs:23:11)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async transcribeFunction (file:///D:/dev/test/brainrot.js/generate/transcribe.mjs:105:31)
at async main (file:///D:/dev/test/brainrot.js/generate/localBuild.mjs:63:2)
at async file:///D:/dev/test/brainrot.js/generate/localBuild.mjs:90:2
audio.mjs file thing
another screeenshot
U need to install ffmpeg. Once u do that make sure you have the flask server on port 5000
like pip install ffmpeg or something?
I ran it and this is what happened: import fetch from 'node-fetch';
import fs from 'fs';
import dotenv from 'dotenv';
import transcriptFunction from './transcript.mjs';
import { writeFile } from 'fs/promises';
import { query } from './dbClient.mjs';
dotenv.config();
import OpenAI from 'openai';
const openai = new OpenAI({
apiKey: process.env.OPENAI_API_KEY,
});
export async function generateTranscriptAudio(
local,
topic,
agentA,
agentB,
ai,
fps,
duration,
background,
music,
videoId
) {
if (!local) {
await query(
"UPDATE pending-videos SET status = 'Generating transcript', progress = 0 WHERE video_id = ?",
[videoId]
);
}
let transcript = (await transcriptFunction(topic, agentA, agentB, duration))
.transcript;
const audios = [];
if (!local) {
await query(
"UPDATE `pending-videos` SET status = 'Fetching images', progress = 5 WHERE video_id = ?",
[videoId]
);
}
const images = await fetchValidImages(
transcript,
transcript.length,
ai,
duration
);
if (!local) {
await query(
"UPDATE `pending-videos` SET status = 'Generating audio', progress = 12 WHERE video_id = ?",
[videoId]
);
}
for (let i = 0; i < transcript.length; i++) {
const person = transcript[i].person;
const line = transcript[i].line;
const voice_id =
person === 'JOE_ROGAN'
? process.env.JOE_ROGAN_VOICE_ID
: person === 'BARACK_OBAMA'
? process.env.BARACK_OBAMA_VOICE_ID
: person === 'BEN_SHAPIRO'
? process.env.BEN_SHAPIRO_VOICE_ID
: person === 'RICK_SANCHEZ'
? process.env.RICK_SANCHEZ_VOICE_ID
: person === 'DONALD_TRUMP'
? process.env.DONALD_TRUMP_VOICE_ID
: person === 'MARK_ZUCKERBERG'
? process.env.MARK_ZUCKERBERG_VOICE_ID
: person === 'JOE_BIDEN'
? process.env.JOE_BIDEN_VOICE_ID
: person === 'LIL_YACHTY'
? process.env.LIL_YACHTY_VOICE_ID
: process.env.JORDAN_PETERSON_VOICE_ID;
await generateAudio(voice_id, person, line, i);
audios.push({
person: person,
audio: `public/voice/${person}-${i}.mp3`,
index: i,
image:
ai && duration === 1
? images[i].imageUrl
: images[i]?.link || 'https://images.smart.wtf/black.png',
});
}
const initialAgentName = audios[0].person;
const contextContent = `
import { staticFile } from 'remotion';
export const music: string = ${
music === 'NONE' ? 'NONE' : '/music/${music}.MP3'
};
export const fps = ${fps};
export const initialAgentName = '${initialAgentName}';
export const videoFileName = '/background/MINECRAFT-9.mp4';
export const subtitlesFileName = [
${audios
.map(
(entry, i) => { name: '${entry.person}', file: staticFile('srt/${entry.person}-${i}.srt'), asset: '${entry.image}', }
)
.join(',\n ')}
];
`;
await writeFile('src/tmp/context.tsx', contextContent, 'utf-8');
return { audios, transcript };
}
export async function generateAudio(voice_id, person, line, index) {
const response = await fetch(
https://api.elevenlabs.io/v1/text-to-speech/${voice_id},
{
method: 'POST',
headers: {
'xi-api-key': process.env.ELEVEN_API_KEY,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model_id: 'eleven_multilingual_v2',
text: line,
voice_settings: {
stability: 0.5,
similarity_boost: 0.75,
},
}),
}
);
if (!response.ok) {
throw new Error(`Server responded with status code ${response.status}`);
}
const audioStream = fs.createWriteStream(
`public/voice/${person}-${index}.mp3`
);
response.body.pipe(audioStream);
return new Promise((resolve, reject) => {
audioStream.on('finish', () => {
resolve('Audio file saved as output.mp3');
});
audioStream.on('error', reject);
});
}
async function fetchValidImages(transcript, length, ai, duration) {
if (ai && duration === 1) {
const promises = [];
for (let i = 0; i < length; i++) {
promises.push(imageGeneneration(transcript[i].asset));
}
const aiImages = await Promise.all(promises);
return aiImages;
} else {
const images = [];
for (let i = 0; i < length; i++) {
const imageFetch = await fetch(
`https://www.googleapis.com/customsearch/v1?q=${encodeURI(
transcript[i].asset
)}&cx=${process.env.GOOGLE_CX}&searchType=image&key=${
process.env.GOOGLE_API_KEY
}&num=${4}`,
{
method: 'GET',
headers: { 'Content-Type': 'application/json' },
}
);
const imageResponse = await imageFetch.json();
// Check if the response contains 'items' and they are iterable
if (
!Array.isArray(imageResponse.items) ||
imageResponse.items.length === 0
) {
console.log(
'No images found or items not iterable',
imageResponse.items
);
images.push({ link: 'https://images.smart.wtf/black.png' });
continue; // Skip to the next iteration
}
const validMimeTypes = ['image/png', 'image/jpeg'];
let imageAdded = false;
for (let image of imageResponse.items) {
if (validMimeTypes.includes(image.mime)) {
const isViewable = await checkImageHeaders(image.link);
if (isViewable) {
images.push(image);
imageAdded = true;
break; // Stop after adding one valid image
}
}
}
// If no valid images were added, push a default image
if (!imageAdded) {
images.push({ link: 'https://images.smart.wtf/black.png' });
}
}
return images;
}
}
async function checkImageHeaders(url) {
try {
const response = await fetch(url, { method: 'HEAD' });
const contentType = response.headers.get('Content-Type');
const contentDisposition = response.headers.get('Content-Disposition');
// Check for direct image content types and absence of attachment disposition
if (
contentType.includes('image/png') ||
contentType.includes('image/jpeg')
) {
if (!contentDisposition || !contentDisposition.includes('attachment')) {
return true; // Image is likely viewable directly in the browser
}
}
} catch (error) {
console.error('Error checking image headers:', error);
}
return false;
}
const imagePrompt = async (title) => {
try {
const response = await openai.chat.completions.create({
model: 'ft:gpt-3.5-turbo-1106:personal::8TEhcfKm',
messages: [
{
role: 'user',
content: title,
},
],
});
return response.choices[0]?.message.content;
} catch (error) {
console.error('Error fetching data:', error);
}
};
const imageGeneneration = async (initialPrompt) => {
const prompt = await imagePrompt(initialPrompt);
const detailed8BitPreface =
'Create an image in a detailed retro 8-bit style. The artwork should have a pixelated texture and should have vibrant coloring and scenery.';
let fullPrompt = `${detailed8BitPreface} ${prompt} Remember, this is in retro 8-bit style`;
fullPrompt = fullPrompt.substring(0, 900);
const responseFetch = await openai.images.generate({
model: 'dall-e-3',
prompt: fullPrompt,
n: 1,
size: '1024x1024',
quality: 'standard',
style: 'vivid',
response_format: 'url',
user: 'user-1234',
});
return {
imageUrl: responseFetch.data[0]?.url,
initialPrompt: initialPrompt,
prompt: prompt,
};
};
$ node localBuild.mjs
file:///D:/dev/test/brainrot.js/generate/eleven.mjs:146
throw new Error(Server responded with status code ${response.status});
^
Error: Server responded with status code 401
at generateAudio (file:///D:/dev/test/brainrot.js/generate/eleven.mjs:146:9)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async generateTranscriptAudio (file:///D:/dev/test/brainrot.js/generate/eleven.mjs:84:3)
at async transcribeFunction (file:///D:/dev/test/brainrot.js/generate/transcribe.mjs:80:33)
at async main (file:///D:/dev/test/brainrot.js/generate/localBuild.mjs:63:2)
at async file:///D:/dev/test/brainrot.js/generate/localBuild.mjs:90:2
Node.js v20.12.2
import transcribeFunction from './transcribe.mjs';
import path from 'path';
import { exec } from 'child_process';
import { topics } from './topics.mjs';
import { rm, mkdir, unlink } from 'fs/promises';
export const PROCESS_ID = 0;
async function cleanupResources() {
try {
await rm(path.join('public', 'srt'), { recursive: true, force: true });
await rm(path.join('public', 'voice'), { recursive: true, force: true });
await unlink(path.join('public', audio-${PROCESS_ID}.mp3)).catch((e) =>
console.error(e)
);
await unlink(path.join('src', 'tmp', 'context.tsx')).catch((e) =>
console.error(e)
);
await mkdir(path.join('public', 'srt'), { recursive: true });
await mkdir(path.join('public', 'voice'), { recursive: true });
} catch (err) {
console.error(Error during cleanup: ${err});
}
}
const agents = [
'BARACK_OBAMA',
'BEN_SHAPIRO',
'JORDAN_PETERSON',
'JOE_ROGAN',
'DONALD_TRUMP',
'MARK_ZUCKERBERG',
'JOE_BIDEN',
'LIL_YACHTY',
// 'RICK_SANCHEZ',
];
const local = true;
async function main() {
const randomTopic = topics[Math.floor(Math.random() * topics.length)];
let agentAIndex = Math.floor(Math.random() * agents.length);
let agentBIndex;
do {
agentBIndex = Math.floor(Math.random() * agents.length);
} while (agentAIndex === agentBIndex);
// CHANGE THIS VALUE FOR YOUR CHOICE OF AGENTS
const agentA = agents[0];
const agentB = agents[4];
// CHANGE THIS VALUE FOR A CUSTOM VIDEO TOPIC
const videoTopic = 'Proximal Policy Optimization';
const aiGeneratedImages = true;
const fps = 15;
const duration = 0.5; //minute
//MINECRAFT or TRUCK or GTA
const background = 'MINECRAFT';
const music = 'NONE';
const cleanSrt = true;
await transcribeFunction(
local,
videoTopic ? videoTopic : randomTopic,
agentA,
agentB,
aiGeneratedImages,
fps,
duration,
background,
music,
cleanSrt
);
// run in the command line `npm run build`
exec('npm run build', async (error, stdout, stderr) => {
if (error) {
console.error(`exec error: ${error}`);
return;
}
console.log(`stdout: ${stdout}`);
console.error(`stderr: ${stderr}`);
cleanupResources();
});
}
(async () => {
await main();
})();
Do you have eleven labs voices and their voice ids in ur .env file? They should be named the same as the voice ids I have (MARK_ZUCKERBERG_VOICE_ID, etc.) which can be found in .env.example