How to implement multipart upload in Node.js? Method introduction
NodeHow to implement multipart upload? The following article will introduce you to the method of implementing multipart upload in Node.js. I hope it will be helpful to you!
#Uploading large files will consume a lot of time, and the upload may fail midway. At this time we need the front-end and back-end to cooperate to solve this problem.
Solution steps:
File fragmentation to reduce the time consumed by each request. If a request fails, you can upload it separately instead of starting from scratch
Notify the server to merge file fragments
Control the number of concurrent requests to avoid browser memory overflow
-
When a request fails due to network or other reasons, we resend the request
File fragmentation and merging
In JavaScript, the FIle object is a subclass of the 'Blob' object, which contains an important method slice, through which we can split the binary file like this:
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Document</title>
<script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script>
</head>
<body>
<input type="file" multiple="multiple" id="fileInput" />
<button onclick="SliceUpload()">上传</button>
<script>
function SliceUpload() {
const file = document.getElementById('fileInput').files[0]
if (!file) return
// 文件分片
let size = 1024 * 50; //50KB 50KB Section size
let fileChunks = [];
let index = 0; //Section num
for (let cur = 0; cur < file.size; cur += size) {
fileChunks.push({
hash: index++,
chunk: file.slice(cur, cur + size),
});
}
// 上传分片
const uploadList = fileChunks.map((item, index) => {
let formData = new FormData();
formData.append("filename", file.name);
formData.append("hash", item.hash);
formData.append("chunk", item.chunk);
return axios({
method: "post",
url: "/upload",
data: formData,
});
});
await Promise.all(uploadList);
// 所有分片上传完成，通知服务器合并分片
await axios({
method: "get",
url: "/merge",
params: {
filename: file.name,
},
});
console.log("Upload to complete");
}
</script>
</body>
</html>Concurrency Control
If the file is large, there will be many fragments divided in this way, and the browser will initiate a large number of requests in a short period of time, which may cause memory exhaustion, so concurrency control is required.
Here we combine the Promise.race() method to control the number of concurrent requests and avoid browser memory overflow.
// 加入并发控制
async function SliceUpload() {
const file = document.getElementById('fileInput').files[0]
if (!file) return
// 文件分片
let size = 1024 * 50; //50KB 50KB Section size
let fileChunks = [];
let index = 0; //Section num
for (let cur = 0; cur < file.size; cur += size) {
fileChunks.push({
hash: index++,
chunk: file.slice(cur, cur + size),
});
}
let pool = []; //Concurrent pool
let max = 3; //Maximum concurrency
for (let i = 0; i < fileChunks.length; i++) {
let item = fileChunks[i];
let formData = new FormData();
formData.append("filename", file.name);
formData.append("hash", item.hash);
formData.append("chunk", item.chunk);
// 上传分片
let task = axios({
method: "post",
url: "/upload",
data: formData,
});
task.then(() => {
// 从并发池中移除已经完成的请求
let index = pool.findIndex((t) => t === task);
pool.splice(index);
});
// 把请求放入并发池中，如果已经达到最大并发量
pool.push(task);
if (pool.length === max) {
//All requests are requested complete
await Promise.race(pool);
}
}
// 所有分片上传完成，通知服务器合并分片
await axios({
method: "get",
url: "/merge",
params: {
filename: file.name,
},
});
console.log("Upload to complete");
}Make code reusable
function SliceUpload() {
const file = document.getElementById('fileInput').files[0]
if (!file) return
// 文件分片
let size = 1024 * 50; // 分片大小设置
let fileChunks = [];
let index = 0; // 分片序号
for (let cur = 0; cur < file.size; cur += size) {
fileChunks.push({
hash: index++,
chunk: file.slice(cur, cur + size),
});
}
const uploadFileChunks = async function(list){
if(list.length === 0){
// 所有分片上传完成，通知如无
await axios({
method: 'get',
url: '/merge',
params: {
filename: file.name
}
});
console.log('Upload to complete')
return
}
let pool = [] // 并发池
let max = 3 // 最大并发数
let finish = 0 // 完成数量
let failList = [] // 失败列表
for(let i=0;i<list.length;i++){
let item = list[i]
let formData = new FormData()
formData.append('filename', file.name)
formData.append('hash', item.hash)
formData.append('chunk', item.chunk)
let task = axios({
method: 'post',
url: '/upload',
data: formData
})
task.then((data)=>{
// 从并发池中移除已经完成的请求
let index = pool.findIndex(t=> t===task)
pool.splice(index)
}).catch(()=>{
failList.push(item)
}).finally(()=>{
finish++
// 如果有失败的重新上传
if(finish===list.length){
uploadFileChunks(failList)
}
})
pool.push(task)
if(pool.length === max){
await Promise.race(pool)
}
}
}
uploadFileChunks(fileChunks)
}Server interface implementation
const express = require('express')
const multiparty = require('multiparty')
const fs = require('fs')
const path = require('path')
const { Buffer } = require('buffer')
// file path
const STATIC_FILES = path.join(__dirname, './static/files')
// Temporary path to upload files
const STATIC_TEMPORARY = path.join(__dirname, './static/temporary')
const server = express()
// Static file hosting
server.use(express.static(path.join(__dirname, './dist')))
// Interface for uploading slices
server.post('/upload', (req, res) => {
const form = new multiparty.Form();
form.parse(req, function(err, fields, files) {
let filename = fields.filename[0]
let hash = fields.hash[0]
let chunk = files.chunk[0]
let dir = `${STATIC_TEMPORARY}/${filename}`
// console.log(filename, hash, chunk)
try {
if (!fs.existsSync(dir)) fs.mkdirSync(dir)
const buffer = fs.readFileSync(chunk.path)
const ws = fs.createWriteStream(`${dir}/${hash}`)
ws.write(buffer)
ws.close()
res.send(`${filename}-${hash} Section uploaded successfully`)
} catch (error) {
console.error(error)
res.status(500).send(`${filename}-${hash} Section uploading failed`)
}
})
})
//Merged slice interface
server.get('/merge', async (req, res) => {
const { filename } = req.query
try {
let len = 0
const bufferList = fs.readdirSync(`${STATIC_TEMPORARY}/${filename}`).map((hash,index) => {
const buffer = fs.readFileSync(`${STATIC_TEMPORARY}/${filename}/${index}`)
len += buffer.length
return buffer
});
//Merge files
const buffer = Buffer.concat(bufferList, len);
const ws = fs.createWriteStream(`${STATIC_FILES}/${filename}`)
ws.write(buffer);
ws.close();
res.send(`Section merge completed`);
} catch (error) {
console.error(error);
}
})
server.listen(3000, _ => {
console.log('http://localhost:3000/')
})For more node-related knowledge, please visit : nodejs tutorial!
The above is the detailed content of How to implement multipart upload in Node.js? Method introduction. For more information, please follow other related articles on the PHP Chinese website!
Hot AI Tools
Undresser.AI Undress
AI-powered app for creating realistic nude photos
AI Clothes Remover
Online AI tool for removing clothes from photos.
Undress AI Tool
Undress images for free
Clothoff.io
AI clothes remover
Video Face Swap
Swap faces in any video effortlessly with our completely free AI face swap tool!
Hot Article
Hot Tools
Notepad++7.3.1
Easy-to-use and free code editor
SublimeText3 Chinese version
Chinese version, very easy to use
Zend Studio 13.0.1
Powerful PHP integrated development environment
Dreamweaver CS6
Visual web development tools
SublimeText3 Mac version
God-level code editing software (SublimeText3)
Hot Topics
Is nodejs a backend framework?
Apr 21, 2024 am 05:09 AM
Node.js can be used as a backend framework as it offers features such as high performance, scalability, cross-platform support, rich ecosystem, and ease of development.
How to connect nodejs to mysql database
Apr 21, 2024 am 06:13 AM
To connect to a MySQL database, you need to follow these steps: Install the mysql2 driver. Use mysql2.createConnection() to create a connection object that contains the host address, port, username, password, and database name. Use connection.query() to perform queries. Finally use connection.end() to end the connection.
What are the global variables in nodejs
Apr 21, 2024 am 04:54 AM
The following global variables exist in Node.js: Global object: global Core module: process, console, require Runtime environment variables: __dirname, __filename, __line, __column Constants: undefined, null, NaN, Infinity, -Infinity
What is the difference between npm and npm.cmd files in the nodejs installation directory?
Apr 21, 2024 am 05:18 AM
There are two npm-related files in the Node.js installation directory: npm and npm.cmd. The differences are as follows: different extensions: npm is an executable file, and npm.cmd is a command window shortcut. Windows users: npm.cmd can be used from the command prompt, npm can only be run from the command line. Compatibility: npm.cmd is specific to Windows systems, npm is available cross-platform. Usage recommendations: Windows users use npm.cmd, other operating systems use npm.
Pi Node Teaching: What is a Pi Node? How to install and set up Pi Node?
Mar 05, 2025 pm 05:57 PM
Detailed explanation and installation guide for PiNetwork nodes This article will introduce the PiNetwork ecosystem in detail - Pi nodes, a key role in the PiNetwork ecosystem, and provide complete steps for installation and configuration. After the launch of the PiNetwork blockchain test network, Pi nodes have become an important part of many pioneers actively participating in the testing, preparing for the upcoming main network release. If you don’t know PiNetwork yet, please refer to what is Picoin? What is the price for listing? Pi usage, mining and security analysis. What is PiNetwork? The PiNetwork project started in 2019 and owns its exclusive cryptocurrency Pi Coin. The project aims to create a one that everyone can participate
Is there a big difference between nodejs and java?
Apr 21, 2024 am 06:12 AM
The main differences between Node.js and Java are design and features: Event-driven vs. thread-driven: Node.js is event-driven and Java is thread-driven. Single-threaded vs. multi-threaded: Node.js uses a single-threaded event loop, and Java uses a multi-threaded architecture. Runtime environment: Node.js runs on the V8 JavaScript engine, while Java runs on the JVM. Syntax: Node.js uses JavaScript syntax, while Java uses Java syntax. Purpose: Node.js is suitable for I/O-intensive tasks, while Java is suitable for large enterprise applications.
Is nodejs a back-end development language?
Apr 21, 2024 am 05:09 AM
Yes, Node.js is a backend development language. It is used for back-end development, including handling server-side business logic, managing database connections, and providing APIs.
Which one to choose between nodejs and java?
Apr 21, 2024 am 04:40 AM
Node.js and Java each have their pros and cons in web development, and the choice depends on project requirements. Node.js excels in real-time applications, rapid development, and microservices architecture, while Java excels in enterprise-grade support, performance, and security.
