基于H5的大文件断点续传系统开发与实现
大型文件分片上传方案设计与实践
系统架构设计
本系统采用前后端分离架构，前端采用Vue2框架实现多终端兼容，后端基于PHP实现服务端逻辑。支持最大10GB文件传输，具备断点续传、加密传输、多云存储适配等功能。
关键技术选型
- 前端框架：Vue2 CLI
- 后端语言：PHP 7.2+
- 数据库：MySQL 8.0
- 云存储：阿里云OSS/腾讯云COS/AWS S3
- 加密算法：SM4/AES
核心模块实现
1. 文件分片处理
export default {
data() {
return {
sliceSize: 5 * 1024 * 1024, // 5MB分片大小
maxConcurrent: 3, // 最大并发数
file: null,
fileId: '',
slices: [],
uploadedSlices: [],
progress: 0
}
},
methods: {
async initializeFileSlicing(file) {
this.file = file
this.fileId = this.generateFileHash(file)
const { data } = await this.$http.get('/upload/progress', {
params: { fileId: this.fileId }
})
if (data.uploadedSlices) {
this.uploadedSlices = data.uploadedSlices
}
const totalSlices = Math.ceil(file.size / this.sliceSize)
this.slices = Array.from({ length: totalSlices }, (_, i) => ({
index: i,
start: i * this.sliceSize,
end: Math.min((i + 1) * this.sliceSize, file.size),
isUploaded: this.uploadedSlices.includes(i)
}))
},
generateFileHash(file) {
return new Promise((resolve) => {
const reader = new FileReader()
reader.onload = (e) => {
const hash = CryptoJS.MD5(e.target.result.slice(0, 1024*1024)).toString()
resolve(hash)
}
reader.readAsArrayBuffer(file.slice(0, 1024*1024))
})
},
async uploadSlice(slice) {
if (slice.isUploaded) return
const blob = this.file.slice(slice.start, slice.end)
const formData = new FormData()
formData.append('fileId', this.fileId)
formData.append('sliceIndex', slice.index)
formData.append('totalSlices', this.slices.length)
formData.append('fileName', this.file.name)
formData.append('fileSize', this.file.size)
formData.append('file', blob)
try {
await this.$http.post('/upload/slice', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
onUploadProgress: (event) => {
this.updateProgress()
}
})
slice.isUploaded = true
this.recordUploadedSlice(slice.index)
} catch (error) {
console.error('分片上传失败:', error)
}
},
async recordUploadedSlice(sliceIndex) {
await this.$http.post('/upload/record', {
fileId: this.fileId,
sliceIndex: sliceIndex
})
},
async mergeFile() {
const { data } = await this.$http.post('/upload/merge', {
fileId: this.fileId,
fileName: this.file.name,
totalSlices: this.slices.length
})
return data.fileUrl
},
updateProgress() {
const completed = this.slices.filter(s => s.isUploaded).length
this.progress = Math.round((completed / this.slices.length) * 100)
}
}
}
2. 文件夹上传处理
export default {
methods: {
handleFolderDrop(event) {
const items = event.dataTransfer.items
if (!items) return
this.processFileTree(items, '', [])
},
processFileTree(items, path = '', fileList = []) {
return new Promise((resolve) => {
const pending = []
for (let i = 0; i < items.length; i++) {
const item = items[i].webkitGetAsEntry()
if (!item) continue
if (item.isFile) {
pending.push(new Promise((res) => {
item.file(file => {
file.relativePath = path + file.name
fileList.push(file)
res()
})
}))
} else if (item.isDirectory) {
const dirReader = item.createReader()
pending.push(new Promise((res) => {
dirReader.readEntries(entries => {
this.processFileTree(entries, `${path}${item.name}/`, fileList)
.then(() => res())
})
}))
}
}
Promise.all(pending).then(() => resolve(fileList))
})
},
async uploadFolder(event) {
const fileList = await this.handleFolderDrop(event)
fileList.forEach(file => {
this.createUploadTask({
file,
relativePath: file.relativePath,
// 其他上传参数...
})
})
}
}
}
3. 加密传输模块
export default {
sm4Encrypt(data, key) {
return CryptoJS.AES.encrypt(data, key).toString()
},
aesEncrypt(data, key, iv) {
const encrypted = CryptoJS.AES.encrypt(
CryptoJS.enc.Utf8.parse(data),
CryptoJS.enc.Utf8.parse(key),
{
iv: CryptoJS.enc.Utf8.parse(iv),
mode: CryptoJS.mode.CBC,
padding: CryptoJS.pad.Pkcs7
}
)
return encrypted.toString()
},
async encryptSlice(slice, config) {
return new Promise((resolve) => {
const reader = new FileReader()
reader.onload = (e) => {
let encryptedData
if (config.type === 'SM4') {
encryptedData = this.sm4Encrypt(
CryptoJS.lib.WordArray.create(e.target.result),
config.key
)
} else if (config.type === 'AES') {
encryptedData = this.aesEncrypt(
CryptoJS.lib.WordArray.create(e.target.result),
config.key,
config.iv
)
}
resolve({
index: slice.index,
data: encryptedData,
encryption: config.type
})
}
reader.readAsArrayBuffer(slice)
})
}
}
服务端实现
1. PHP处理逻辑
class FileUploadHandler {
public function checkUploadStatus($fileId) {
$uploadedSlices = $this->db->query(
"SELECT slice_index FROM file_slices WHERE file_id = ?",
[$fileId]
)->fetchAll(PDO::FETCH_COLUMN);
return [
'uploadedSlices' => $uploadedSlices,
'serverConfig' => [
'sliceSize' => 5 * 1024 * 1024,
'maxConcurrent' => 3
]
];
}
public function recordSlice($fileId, $sliceIndex) {
try {
$this->db->beginTransaction();
$exists = $this->db->query(
"SELECT 1 FROM file_slices WHERE file_id = ? AND slice_index = ?",
[$fileId, $sliceIndex]
)->fetchColumn();
if (!$exists) {
$this->db->prepare(
"INSERT INTO file_slices (file_id, slice_index, upload_time)
VALUES (?, ?, NOW())"
)->execute([$fileId, $sliceIndex]);
}
$this->db->commit();
return ['success' => true];
} catch (Exception $e) {
$this->db->rollBack();
return ['success' => false, 'error' => $e->getMessage()];
}
}
public function mergeFile($fileId, $fileName, $totalSlices) {
$tempDir = sys_get_temp_dir() . '/uploads/' . $fileId;
$finalPath = 'uploads/' . uniqid() . '_' . $fileName;
if (!is_dir($tempDir)) {
throw new Exception("临时目录不存在");
}
$finalFile = fopen($finalPath, 'wb');
for ($i = 0; $i < $totalSlices; $i++) {
$slicePath = $tempDir . '/' . $i;
if (!is_file($slicePath)) {
throw new Exception("缺少分片: " . $i);
}
$sliceData = file_get_contents($slicePath);
fwrite($finalFile, $sliceData);
}
fclose($finalFile);
$this->removeTempDirectory($tempDir);
$this->saveFileInfo($fileId, $fileName, $finalPath);
return ['fileUrl' => $finalPath];
}
private function removeTempDirectory($dir) {
if (is_dir($dir)) {
$objects = scandir($dir);
foreach ($objects as $object) {
if ($object != "." && $object != "..") {
if (is_dir($dir . "/" . $object)) {
$this->removeTempDirectory($dir . "/" . $object);
} else {
unlink($dir . "/" . $object);
}
}
}
rmdir($dir);
}
}
}
2. 数据库设计
-- 文件上传记录表
CREATE TABLE `file_uploads` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`file_id` varchar(64) NOT NULL COMMENT '文件唯一标识',
`file_name` varchar(255) NOT NULL COMMENT '原始文件名',
`file_size` bigint(20) NOT NULL COMMENT '文件大小(字节)',
`total_slices` int(11) NOT NULL COMMENT '总分片数',
`user_id` int(11) DEFAULT NULL COMMENT '上传用户ID',
`status` enum('uploading','completed','failed') NOT NULL DEFAULT 'uploading',
`created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_file_id` (`file_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- 分片存储记录表
CREATE TABLE `file_slices` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`file_id` varchar(64) NOT NULL COMMENT '关联的文件ID',
`slice_index` int(11) NOT NULL COMMENT '分片索引',
`upload_time` datetime NOT NULL COMMENT '上传时间',
`storage_path` varchar(512) DEFAULT NULL COMMENT '存储路径(临时)',
PRIMARY KEY (`id`),
UNIQUE KEY `idx_file_slice` (`file_id`,`slice_index`),
KEY `idx_file_id` (`file_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- 完成文件记录表
CREATE TABLE `completed_files` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`file_id` varchar(64) NOT NULL,
`storage_path` varchar(512) NOT NULL COMMENT '最终存储路径',
`storage_provider` varchar(32) NOT NULL COMMENT '存储提供商',
`file_url` varchar(512) NOT NULL COMMENT '访问URL',
`encryption` varchar(16) DEFAULT NULL COMMENT '加密方式',
`metadata` text COMMENT '文件元数据(JSON)',
`created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_file_id` (`file_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
兼容性方案
if (window.navigator.userAgent.indexOf('MSIE 8.0') > -1) {
// 添加polyfill
if (!window.JSON) {
document.write('<\/script>')
}
// 重写FileReader API
if (!window.FileReader) {
window.FileReader = function() {
this.readAsArrayBuffer = function(file) {
console.warn('IE8不支持FileReader，请使用Flash上传组件')
}
}
}
// 添加FormData兼容
if (!window.FormData) {
window.FormData = function() {
this.data = {}
this.append = function(name, value) {
this.data[name] = value
}
}
}
console.log('检测到IE8浏览器，部分功能可能受限，建议使用现代浏览器')
}
性能优化策略
- 动态调整分片大小：根据网络状况自动优化分片尺寸
- 智能并发控制：根据浏览器性能限制上传线程数
- 本地缓存机制：使用IndexedDB记录上传状态
- 长连接维持：实现心跳机制防止超时
- 内存优化：使用流式处理减少内存占用
云存储适配
const CloudStorage = {
providers: {
aliyun: {
upload(file, config) {
return new Promise((resolve, reject) => {
const client = new OSS({
region: config.region,
accessKeyId: config.accessKeyId,
accessKeySecret: config.accessKeySecret,
bucket: config.bucket
})
client.multipartUpload(config.key, file, {
progress: (p) => {
config.onProgress(p * 100)
}
}).then(resolve).catch(reject)
})
}
},
aws: {
upload(file, config) {
const s3 = new AWS.S3({
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
region: config.region
})
const params = {
Bucket: config.bucket,
Key: config.key,
Body: file,
ContentType: file.type
}
return s3.upload(params, {
partSize: config.chunkSize || 5 * 1024 * 1024,
queueSize: config.concurrent || 3
}).promise()
}
}
},
upload(provider, file, config) {
if (!this.providers[provider]) {
throw new Error(`不支持的云存储提供商: ${provider}`)
}
return this.providers[provider].upload(file, config)
}
}
