failed to replicate Large document Attachments over WebRTC in browsers

[haouarihk]

Have you found a bug?

WebRTC in browsers is only limited to 16kb,
however this limit is not present in nodejs environment.

Did you make a unit test?

As far as i can tell, there are no browser based unit test in this project, but i made the test anyway for future implementation.

Do you have a feature request?

The solution to this problem is to send large contents in chunks, rather than sending them all as one blob. this could also open us for another event listener for "syncing".

if i missed anything, be sure to let me know.

PR: #6521

[haouarihk]

here's the test file:

import assert from 'assert';
import config from './config.ts';
import {
    randomCouchString,
    createRxDatabase
} from '../../plugins/core/index.mjs';
import {
    replicateWebRTC,
    getConnectionHandlerSimplePeer,
    SimplePeerWrtc
} from '../../plugins/replication-webrtc/index.mjs';
import {
    isFastMode,
    isDeno,
    isNode
} from '../../plugins/test-utils/index.mjs';
import { waitUntil } from 'async-test-util';
import { wrappedAttachmentsCompressionStorage } from '../../plugins/attachments-compression/index.mjs';

describe('large-document-webrtc.test.ts', function () {
    this.timeout(1000 * 60); // 1 minute

    if (
        !config.storage.hasReplication ||
        !config.storage.hasPersistence ||
        !config.storage.hasAttachments ||
        isDeno
    ) {
        return;
    }

    let wrtc: SimplePeerWrtc;
    let webSocketConstructor: WebSocket;
    const signalingServerUrl = 'ws://localhost:18006';

    before(async () => {
        if (isNode) {
            const wrtcModule = await import('node-datachannel/polyfill');
            wrtc = wrtcModule.default as any;

            const wsModule = await import('ws');
            webSocketConstructor = wsModule.WebSocket as any;
        }
    });

    it('should replicate large attachments over WebRTC', async function () {
        if (isFastMode()) {
            return;
        }

        const createCollection = async (name: string) => {
            const db = await createRxDatabase({
                name: randomCouchString(10),
                storage: wrappedAttachmentsCompressionStorage({
                    storage: config.storage.getStorage()
                })
            });

            const collections = await db.addCollections({
                [name]: {
                    schema: {
                        version: 0,
                        type: 'object',
                        primaryKey: 'id',
                        properties: {
                            id: {
                                type: 'string'
                            },
                            name: {
                                type: 'string'
                            }
                        },
                        attachments: {
                            compression: 'gzip'
                        }
                    }
                }
            });

            return collections[name];
        };

        const c1 = await createCollection('humans1');
        const c2 = await createCollection('humans2');

        const topic = randomCouchString(10);

        const replicationStates = await Promise.all([c1, c2].map(collection =>
            replicateWebRTC({
                collection,
                topic,
                connectionHandlerCreator: getConnectionHandlerSimplePeer({
                    signalingServerUrl,
                    wrtc,
                    webSocketConstructor
                }),
                pull: {},
                push: {}
            })
        ));

        // Create a large attachment (1GB)
        const largeData = new Uint8Array(1 * 1024 * 1024 * 1024);
        for (let i = 0; i < largeData.length; i++) {
            largeData[i] = Math.floor(Math.random() * 256);
        }

        const doc = await c1.insert({
            id: 'doc1',
            name: 'Large Document'
        });

        await doc.putAttachment({
            id: 'large-attachment',
            data: Buffer.from(largeData),
            type: 'application/octet-stream'
        });

        // Wait for replication
        await waitUntil(async () => {
            const docInC2 = await c2.findOne('doc1').exec();
            return docInC2 && docInC2.allAttachments().length > 0;
        });

        const replicatedDoc = await c2.findOne('doc1').exec();
        assert.ok(replicatedDoc, 'Document should be replicated');

        const attachment = replicatedDoc.getAttachment('large-attachment');
        assert.ok(attachment, 'Attachment should be replicated');

        const attachmentData = await attachment.getData();
        const replicatedData = new Uint8Array(await attachmentData.arrayBuffer());

        assert.strictEqual(replicatedData.length, largeData.length, 'Attachment size should match');
        assert.deepStrictEqual(replicatedData, largeData, 'Attachment content should match');

        // Clean up
        await Promise.all(replicationStates.map(state => state.cancel()));
        await c1.database.destroy();
        await c2.database.destroy();
    });
});

[stale]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed soon. Please update it or it may be closed to keep our repository organized. The best way is to add some more information or make a pull request with a test case. Also you might get help in fixing it at the RxDB Community Chat If you know you will continue working on this, just write any message to the issue (like "ping") to remove the stale tag.