theophilusx/ssh2-sftp-client

Slow download speed

JPVRS opened this issue · 2 comments

JPVRS commented

I'm running into this issue as I try to script the downloading of a roughly 1gb file from a remote SFTP host. The file downloads in roughly 1 minute using WinSCP. The script is successfully connecting and starting the download, but the download speed is something like 10mb per minute. I'm sure I'm doing something wrong, but not sure what. I've tried playing around with the highWaterMark and concurrency settings (and also not using them at all) and the result doesn't change. I'm currently running this on a Windows 10 machine.

Update: I've resolved this by switching to using SCP for the download. It's possible I was just choosing the wrong tool for the job.

require("dotenv").config();
const SftpClient = require("ssh2-sftp-client");
const AdmZip = require("adm-zip");
const os = require("os");
const path = require("path");
const fs = require("fs");

// Constants for remote paths
const REMOTE_PATH = "./remote/path";

// Directory where the zip archives will be downloaded and extracted
const downloadDirectory = path.join(
  os.homedir(),
  "local",
  "path",
);
const extractedFilesDirectory = path.join(downloadDirectory, "extracted-files");

// Ensure the download and extraction directories exist
const ensureDirectoriesExist = () => {
  if (!fs.existsSync(downloadDirectory)) {
    fs.mkdirSync(downloadDirectory, { recursive: true });
    console.log(`Created download directory: ${downloadDirectory}`);
  }

  if (!fs.existsSync(extractedFilesDirectory)) {
    fs.mkdirSync(extractedFilesDirectory, { recursive: true });
    console.log(`Created extraction directory: ${extractedFilesDirectory}`);
  }
};

// Connect to the SFTP server and list zip archives
const listArchives = async () => {
  const client = new SftpClient();
  const host = process.env.FTP_HOST;
  const port = process.env.FTP_PORT;
  const username = process.env.FTP_USER_NAME;
  const password = process.env.FTP_PASSWORD;

  try {
    console.log(`Connecting to SFTP server at ${host}:${port}...`);
    await client.connect({ host, port, username, password });
    console.log("Connected to SFTP server.");
    console.log(`Navigating to remote folder: ${REMOTE_PATH}`);

    const fileList = await client.list(REMOTE_PATH);
    const zipFiles = fileList
      .filter((file) => file.type === "-" && file.name.endsWith(".zip"))
      .sort((a, b) => new Date(a.modifyTime) - new Date(b.modifyTime));

    console.log("Zip archives found (oldest to newest):");
    zipFiles.forEach((file) => console.log(file.name));

    return { client, zipFiles };
  } catch (err) {
    console.error("Failed to connect to SFTP server or list archives:", err);
    client.end();
    process.exit(1);
  }
};

// Prompt user to select an archive to download
const selectArchiveToDownload = async (zipFiles) => {
  const { default: inquirer } = await import("inquirer");
  const choices = zipFiles.map((file) => file.name);
  const questions = [
    {
      type: "list",
      name: "selectedArchive",
      message: "Select a zip archive to download:",
      choices: choices,
    },
  ];

  const answers = await inquirer.prompt(questions);
  return answers.selectedArchive;
};

// Download and extract the selected archive
const downloadAndExtractArchive = async (client, selectedArchive) => {
  const localPath = path.join(downloadDirectory, selectedArchive);
  const remoteFilePath = path.join(REMOTE_PATH, selectedArchive);

  try {
    console.log(`Downloading ${selectedArchive} to ${localPath}...`);
    await client.fastGet(remoteFilePath, localPath, {
      highWaterMark: 2048576,
      concurrency: 8,
    });

    console.log(`Downloaded ${selectedArchive}.`);

    console.log(`Extracting ${selectedArchive}...`);
    const zip = new AdmZip(localPath);
    zip.extractAllTo(extractedFilesDirectory, true);
    console.log(`Extracted to ${extractedFilesDirectory}.`);
  } catch (err) {
    console.error("Failed to download or extract archive:", err);
    process.exit(1);
  } finally {
    client.end();
  }
};

// Main script function
const main = async () => {
  ensureDirectoriesExist();
  const { client, zipFiles } = await listArchives();

  if (zipFiles.length === 0) {
    console.log("No zip archives to download.");
    client.end();
    return;
  }

  const selectedArchive = await selectArchiveToDownload(zipFiles);
  await downloadAndExtractArchive(client, selectedArchive);
};

main();
theophilusx commented
JPVRS commented

Thank you for the response. Next time I'll try simply using get() and the other steps you suggested. Please feel free to close.