Skip to content

Commit a1dad3e

Browse files
carsonxucarsonxu
committed
修改队列清理逻辑
1 parent 6ffee7f commit a1dad3e

File tree

7 files changed

+772
-721
lines changed

7 files changed

+772
-721
lines changed

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "cos-nodejs-sdk-v5",
3-
"version": "2.5.6",
3+
"version": "2.5.7",
44
"description": "cos nodejs sdk v5",
55
"main": "index.js",
66
"scripts": {

‎sdk/advance.js‎

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@ function getUploadIdAndPartList(params, callback) {
353353
Bucket: Bucket,
354354
Region: Region,
355355
Key: Key,
356-
Headers: params.Headers,
356+
Headers: util.clone(params.Headers),
357357
StorageClass: StorageClass,
358358
}, params);
359359
self.multipartInit(_params, function (err, data) {
@@ -519,7 +519,7 @@ function wholeMultipartList(params, callback) {
519519
self.multipartList(sendParams, function (err, data) {
520520
if (err) return callback(err);
521521
UploadList.push.apply(UploadList, data.Upload || []);
522-
if (data.IsTruncated == 'true') { // 列表不完整
522+
if (data.IsTruncated === 'true') { // 列表不完整
523523
sendParams.KeyMarker = data.NextKeyMarker;
524524
sendParams.UploadIdMarker = data.NextUploadIdMarker;
525525
next();
@@ -545,7 +545,7 @@ function wholeMultipartListPart(params, callback) {
545545
self.multipartListPart(sendParams, function (err, data) {
546546
if (err) return callback(err);
547547
PartList.push.apply(PartList, data.Part || []);
548-
if (data.IsTruncated == 'true') { // 列表不完整
548+
if (data.IsTruncated === 'true') { // 列表不完整
549549
sendParams.PartNumberMarker = data.NextPartNumberMarker;
550550
next();
551551
} else {
@@ -814,10 +814,10 @@ function abortUploadTaskArray(params, callback) {
814814
var resultList = new Array(AbortArray.length);
815815
Async.eachLimit(AbortArray, AsyncLimit, function (AbortItem, callback) {
816816
var eachIndex = index;
817-
if (Key && Key != AbortItem.Key) {
818-
return callback(null, {
819-
KeyNotMatch: true
820-
});
817+
if (Key && Key !== AbortItem.Key) {
818+
resultList[eachIndex] = {error: {KeyNotMatch: true}};
819+
callback(null);
820+
return;
821821
}
822822
var UploadId = AbortItem.UploadId || AbortItem.UploadID;
823823

@@ -1008,15 +1008,15 @@ function sliceCopyFile(params, callback) {
10081008
var currentSize = SliceItem.end - SliceItem.start;
10091009
var preAddSize = 0;
10101010

1011-
copySliceItem.call(self,{
1011+
copySliceItem.call(self, {
10121012
Bucket: Bucket,
10131013
Region: Region,
10141014
Key: Key,
10151015
CopySource: CopySource,
1016-
UploadId:UploadData.UploadId,
1017-
PartNumber:PartNumber,
1018-
CopySourceRange:CopySourceRange,
1019-
onProgress:function (data) {
1016+
UploadId: UploadData.UploadId,
1017+
PartNumber: PartNumber,
1018+
CopySourceRange: CopySourceRange,
1019+
onProgress: function (data) {
10201020
FinishSize += data.loaded - preAddSize;
10211021
preAddSize = data.loaded;
10221022
onProgress({loaded: FinishSize, total: FileSize});
@@ -1041,7 +1041,7 @@ function sliceCopyFile(params, callback) {
10411041
});
10421042
});
10431043

1044-
ep.on('get_file_size_finish',function () {
1044+
ep.on('get_file_size_finish', function (SourceHeaders) {
10451045
// 控制分片大小
10461046
(function () {
10471047
var SIZE = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1024 * 2, 1024 * 4, 1024 * 5];
@@ -1069,10 +1069,19 @@ function sliceCopyFile(params, callback) {
10691069
params.PartList = list;
10701070
})();
10711071

1072+
var TargetHeader;
1073+
if (params.Headers['x-cos-metadata-directive'] === 'Replaced') {
1074+
TargetHeader = params.Headers;
1075+
} else {
1076+
TargetHeader = SourceHeaders;
1077+
}
1078+
TargetHeader['x-cos-storage-class'] = params.Headers['x-cos-storage-class'] || SourceHeaders['x-cos-storage-class'];
1079+
TargetHeader = util.clearKey(TargetHeader);
10721080
self.multipartInit({
10731081
Bucket: Bucket,
10741082
Region: Region,
10751083
Key: Key,
1084+
Headers: TargetHeader,
10761085
},function (err,data) {
10771086
if (err) {
10781087
return callback(err);
@@ -1098,10 +1107,18 @@ function sliceCopyFile(params, callback) {
10981107
}
10991108

11001109
FileSize = params.FileSize = data.headers['content-length'];
1110+
if (FileSize === undefined || !FileSize) {
1111+
callback({error: 'get Content-Length error, please add "Content-Length" to CORS ExposeHeader setting.'});
1112+
return;
1113+
}
1114+
11011115
onProgress = util.throttleOnProgress.call(self, FileSize, params.onProgress);
11021116

11031117
// 开始上传
11041118
if (FileSize <= CopySliceSize) {
1119+
if (!params.Headers['x-cos-metadata-directive']) {
1120+
params.Headers['x-cos-metadata-directive'] = 'Copy';
1121+
}
11051122
self.putObjectCopy(params, function (err, data) {
11061123
if (err) {
11071124
onProgress(null, true);
@@ -1111,7 +1128,22 @@ function sliceCopyFile(params, callback) {
11111128
callback(err, data);
11121129
});
11131130
} else {
1114-
ep.emit('get_file_size_finish');
1131+
var resHeaders = data.headers;
1132+
var SourceHeaders = {
1133+
'Cache-Control': resHeaders['cache-control'],
1134+
'Content-Disposition': resHeaders['content-disposition'],
1135+
'Content-Encoding': resHeaders['content-encoding'],
1136+
'Content-Type': resHeaders['content-type'],
1137+
'Expires': resHeaders['expires'],
1138+
'x-cos-storage-class': resHeaders['x-cos-storage-class'],
1139+
};
1140+
util.each(resHeaders, function (v, k) {
1141+
var metaPrefix = 'x-cos-meta-';
1142+
if (k.indexOf(metaPrefix) === 0 && k.length > metaPrefix.length) {
1143+
SourceHeaders[k] = v;
1144+
}
1145+
});
1146+
ep.emit('get_file_size_finish', SourceHeaders);
11151147
}
11161148
});
11171149
}

‎sdk/base.js‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1392,7 +1392,7 @@ function optionsObject(params, callback) {
13921392
* @param {String} CopySourceIfUnmodifiedSince 当Object在指定时间后未被修改，则执行操作，否则返回412。可与x-cos-copy-source-If-Match一起使用，与其他条件联合使用返回冲突。
13931393
* @param {String} CopySourceIfMatch 当Object的Etag和给定一致时，则执行操作，否则返回412。可与x-cos-copy-source-If-Unmodified-Since一起使用，与其他条件联合使用返回冲突。
13941394
* @param {String} CopySourceIfNoneMatch 当Object的Etag和给定不一致时，则执行操作，否则返回412。可与x-cos-copy-source-If-Modified-Since一起使用，与其他条件联合使用返回冲突。
1395-
* @param {String} StorageClass 存储级别，枚举值：存储级别，枚举值：Standard, Standard_IA，Nearline；默认值：Standard
1395+
* @param {String} StorageClass 存储级别，枚举值：存储级别，枚举值：Standard, Standard_IA，Archive；默认值：Standard
13961396
* @param {String} CacheControl 指定所有缓存机制在整个请求/响应链中必须服从的指令。
13971397
* @param {String} ContentDisposition MIME 协议的扩展，MIME 协议指示 MIME 用户代理如何显示附加的文件
13981398
* @param {String} ContentEncoding HTTP 中用来对「采用何种编码格式传输正文」进行协定的一对头部字段
@@ -1593,7 +1593,7 @@ function restoreObject(params, callback) {
15931593
* @param {String} params.GrantRead 赋予被授权者读的权限 ，非必须
15941594
* @param {String} params.GrantWrite 赋予被授权者写的权限 ，非必须
15951595
* @param {String} params.GrantFullControl 赋予被授权者读写权限 ，非必须
1596-
* @param {String} params.StorageClass 设置Object的存储级别，枚举值：Standard，Standard_IA，Nearline，非必须
1596+
* @param {String} params.StorageClass 设置Object的存储级别，枚举值：Standard，Standard_IA，Archive，非必须
15971597
* @param {String} params.ServerSideEncryption 支持按照指定的加密算法进行服务端数据加密，格式 x-cos-server-side-encryption: "AES256"，非必须
15981598
* @param {Function} callback 回调函数，必须
15991599
* @return {Object} err 请求失败的错误，如果请求成功，则为空。https://cloud.tencent.com/document/product/436/7730
@@ -2099,6 +2099,13 @@ function getUrl(params) {
20992099
// 异步获取签名
21002100
function getAuthorizationAsync(params, callback) {
21012101

2102+
var headers = util.clone(params.Headers);
2103+
delete headers['Content-Type'];
2104+
delete headers['Cache-Control'];
2105+
util.each(headers, function (v, k) {
2106+
v === '' && delete headers[k];
2107+
});
2108+
21022109
var cb = function (AuthData) {
21032110

21042111
// 检查签名格式
@@ -2184,7 +2191,7 @@ function getAuthorizationAsync(params, callback) {
21842191
Method: params.Method,
21852192
Pathname: Pathname,
21862193
Query: params.Query,
2187-
Headers: params.Headers,
2194+
Headers: headers,
21882195
UseRawKey: self.options.UseRawKey,
21892196
SystemClockOffset: self.options.SystemClockOffset,
21902197
});
@@ -2209,7 +2216,7 @@ function getAuthorizationAsync(params, callback) {
22092216
Key: KeyName,
22102217
Pathname: Pathname,
22112218
Query: params.Query,
2212-
Headers: params.Headers,
2219+
Headers: headers,
22132220
Scope: Scope,
22142221
}, function (AuthData) {
22152222
if (typeof AuthData === 'string') {
@@ -2249,7 +2256,7 @@ function getAuthorizationAsync(params, callback) {
22492256
Method: params.Method,
22502257
Pathname: Pathname,
22512258
Query: params.Query,
2252-
Headers: params.Headers,
2259+
Headers: headers,
22532260
Expires: params.Expires,
22542261
UseRawKey: self.options.UseRawKey,
22552262
SystemClockOffset: self.options.SystemClockOffset,
@@ -2566,15 +2573,15 @@ function _submitRequest(params, callback) {
25662573
if (readStream) {
25672574
readStream.on('error', function (err) {
25682575
sender && sender.abort && sender.abort();
2569-
callback(err);
2576+
cb(err);
25702577
});
25712578
readStream.pipe(sender);
25722579
}
25732580
// pipe 输出
25742581
if (params.outputStream) {
25752582
params.outputStream.on('error', function (err) {
25762583
sender && sender.abort && sender.abort();
2577-
callback(err)
2584+
cb(err)
25782585
});
25792586
sender.pipe(params.outputStream);
25802587
}

‎sdk/cos.js‎

Lines changed: 68 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,68 @@
1-
'use strict';
2-
3-
var util = require('./util');
4-
var event = require('./event');
5-
var task = require('./task');
6-
var base = require('./base');
7-
var advance = require('./advance');
8-
var pkg = require('../package.json');
9-
10-
var defaultOptions = {
11-
AppId: '', // AppId 已废弃，请拼接到 Bucket 后传入，例如：test-1250000000
12-
SecretId: '',
13-
SecretKey: '',
14-
XCosSecurityToken: '', // 使用临时密钥需要注意自行刷新 Token
15-
FileParallelLimit: 3,
16-
ChunkParallelLimit: 3,
17-
ChunkRetryTimes: 3,
18-
ChunkSize: 1024 * 1024,
19-
SliceSize: 1024 * 1024,
20-
CopyChunkParallelLimit: 20,
21-
CopyChunkSize: 1024 * 1024 * 10,
22-
CopySliceSize: 1024 * 1024 * 10,
23-
MaxPartNumber: 10000,
24-
ProgressInterval: 1000,
25-
UploadQueueSize: 10000,
26-
Domain: '',
27-
ServiceDomain: '',
28-
Protocol: '',
29-
CompatibilityMode: false,
30-
ForcePathStyle: false,
31-
UseRawKey: false,
32-
Timeout: 0, // 单位毫秒，0 代表不设置超时时间
33-
CorrectClockSkew: true,
34-
SystemClockOffset: 0, // 单位毫秒，ms
35-
UploadCheckContentMd5: false,
36-
UploadIdCacheLimit: 500,
37-
Proxy: '',
38-
Ip: '',
39-
StrictSsl: true,
40-
UserAgent: '',
41-
};
42-
43-
// 对外暴露的类
44-
var COS = function (options) {
45-
this.options = util.extend(util.clone(defaultOptions), options || {});
46-
this.options.FileParallelLimit = Math.max(1, this.options.FileParallelLimit);
47-
this.options.ChunkParallelLimit = Math.max(1, this.options.ChunkParallelLimit);
48-
this.options.ChunkRetryTimes = Math.max(0, this.options.ChunkRetryTimes);
49-
this.options.ChunkSize = Math.max(1024 * 1024, this.options.ChunkSize);
50-
this.options.CopyChunkParallelLimit = Math.max(1, this.options.CopyChunkParallelLimit);
51-
this.options.CopyChunkSize = Math.max(1024 * 1024, this.options.CopyChunkSize);
52-
this.options.CopySliceSize = Math.max(0, this.options.CopySliceSize);
53-
this.options.MaxPartNumber = Math.max(1024, Math.min(10000, this.options.MaxPartNumber));
54-
this.options.Timeout = Math.max(0, this.options.Timeout);
55-
if (this.options.AppId) {
56-
console.warn('warning: AppId has been deprecated, Please put it at the end of parameter Bucket(E.g: "test-1250000000").');
57-
}
58-
event.init(this);
59-
task.init(this);
60-
};
61-
62-
base.init(COS, task);
63-
advance.init(COS, task);
64-
65-
COS.getAuthorization = util.getAuth;
66-
COS.version = pkg.version;
67-
68-
module.exports = COS;
1+
'use strict';
2+
3+
var util = require('./util');
4+
var event = require('./event');
5+
var task = require('./task');
6+
var base = require('./base');
7+
var advance = require('./advance');
8+
var pkg = require('../package.json');
9+
10+
var defaultOptions = {
11+
AppId: '', // AppId 已废弃，请拼接到 Bucket 后传入，例如：test-1250000000
12+
SecretId: '',
13+
SecretKey: '',
14+
XCosSecurityToken: '', // 使用临时密钥需要注意自行刷新 Token
15+
FileParallelLimit: 3,
16+
ChunkParallelLimit: 3,
17+
ChunkRetryTimes: 3,
18+
ChunkSize: 1024 * 1024,
19+
SliceSize: 1024 * 1024,
20+
CopyChunkParallelLimit: 20,
21+
CopyChunkSize: 1024 * 1024 * 10,
22+
CopySliceSize: 1024 * 1024 * 10,
23+
MaxPartNumber: 10000,
24+
ProgressInterval: 1000,
25+
UploadQueueSize: 1000,
26+
Domain: '',
27+
ServiceDomain: '',
28+
Protocol: '',
29+
CompatibilityMode: false,
30+
ForcePathStyle: false,
31+
UseRawKey: false,
32+
Timeout: 0, // 单位毫秒，0 代表不设置超时时间
33+
CorrectClockSkew: true,
34+
SystemClockOffset: 0, // 单位毫秒，ms
35+
UploadCheckContentMd5: false,
36+
UploadIdCacheLimit: 500,
37+
Proxy: '',
38+
Ip: '',
39+
StrictSsl: true,
40+
UserAgent: '',
41+
};
42+
43+
// 对外暴露的类
44+
var COS = function (options) {
45+
this.options = util.extend(util.clone(defaultOptions), options || {});
46+
this.options.FileParallelLimit = Math.max(1, this.options.FileParallelLimit);
47+
this.options.ChunkParallelLimit = Math.max(1, this.options.ChunkParallelLimit);
48+
this.options.ChunkRetryTimes = Math.max(0, this.options.ChunkRetryTimes);
49+
this.options.ChunkSize = Math.max(1024 * 1024, this.options.ChunkSize);
50+
this.options.CopyChunkParallelLimit = Math.max(1, this.options.CopyChunkParallelLimit);
51+
this.options.CopyChunkSize = Math.max(1024 * 1024, this.options.CopyChunkSize);
52+
this.options.CopySliceSize = Math.max(0, this.options.CopySliceSize);
53+
this.options.MaxPartNumber = Math.max(1024, Math.min(10000, this.options.MaxPartNumber));
54+
this.options.Timeout = Math.max(0, this.options.Timeout);
55+
if (this.options.AppId) {
56+
console.warn('warning: AppId has been deprecated, Please put it at the end of parameter Bucket(E.g: "test-1250000000").');
57+
}
58+
event.init(this);
59+
task.init(this);
60+
};
61+
62+
base.init(COS, task);
63+
advance.init(COS, task);
64+
65+
COS.getAuthorization = util.getAuth;
66+
COS.version = pkg.version;
67+
68+
module.exports = COS;

‎sdk/task.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@ var initTask = function (cos) {
5252
queue.length > cos.options.UploadQueueSize && // 大于队列才处理
5353
i < nextUploadIndex; // 小于当前操作的 index 才处理
5454
i++) {
55-
if (!queue[i] || queue[i].state !== 'waiting') {
55+
var isActive = queue[i].state === 'waiting' || queue[i].state === 'checking' || queue[i].state === 'uploading';
56+
if (!queue[i] || !isActive) {
57+
tasks[queue[i].id] && (delete tasks[queue[i].id]);
5658
queue.splice(i, 1);
5759
nextUploadIndex--;
5860
}
@@ -64,6 +66,7 @@ var initTask = function (cos) {
6466
if (nextUploadIndex < queue.length &&
6567
uploadingFileCount < cos.options.FileParallelLimit) {
6668
var task = queue[nextUploadIndex];
69+
nextUploadIndex++;
6770
if (task.state === 'waiting') {
6871
uploadingFileCount++;
6972
task.state = 'checking';
@@ -91,7 +94,6 @@ var initTask = function (cos) {
9194
});
9295
emitListUpdate();
9396
}
94-
nextUploadIndex++;
9597
startNextTask(cos);
9698
}
9799
};
@@ -221,6 +223,9 @@ var initTask = function (cos) {
221223
startNextTask();
222224
}
223225
};
226+
cos.isUploadRunning = function () {
227+
return uploadingFileCount || nextUploadIndex < queue.length;
228+
};
224229

225230
};
226231

0 commit comments

Comments
 (0)