Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adding a range download, a verification with a status code of 206 is … #514

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

adding a range download, a verification with a status code of 206 is … #514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ee397c
adding a range download, a verification with a status code of 206 is …
zhuxiaolong37 Jan 23, 2024
b145d91
Ensure consistency verification when uploading multiple open files wi…
zhuxiaolong37 Mar 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 32 additions & 2 deletions src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import com.aliyun.oss.ClientException;
import com.aliyun.oss.InconsistentException;
import com.aliyun.oss.common.utils.BinaryUtil;
import com.aliyun.oss.common.utils.CRC64;
Expand Down Expand Up @@ -539,11 +540,13 @@ private DownloadResult download(DownloadCheckPoint downloadCheckPoint, DownloadF
ProgressPublisher.publishResponseBytesTransferred(listener, completedLength);
downloadFileRequest.setProgressListener(null);

SimplifiedObjectMeta meta = objectOperation.getSimplifiedObjectMeta(new GenericRequest(downloadFileRequest.getBucketName(), downloadFileRequest.getKey()));

// Concurrently download parts.
for (int i = 0; i < downloadCheckPoint.downloadParts.size(); i++) {
if (!downloadCheckPoint.downloadParts.get(i).isCompleted) {
Task task = new Task(i, "download-" + i, downloadCheckPoint, i, downloadFileRequest, objectOperation,
listener);
listener, meta);
futures.add(service.submit(task));
tasks.add(task);
} else {
Expand Down Expand Up @@ -592,14 +595,15 @@ class Task implements Callable<PartResult> {

public Task(int id, String name, DownloadCheckPoint downloadCheckPoint, int partIndex,
DownloadFileRequest downloadFileRequest, OSSObjectOperation objectOperation,
ProgressListener progressListener) {
ProgressListener progressListener, SimplifiedObjectMeta simplifiedMeta) {
this.id = id;
this.name = name;
this.downloadCheckPoint = downloadCheckPoint;
this.partIndex = partIndex;
this.downloadFileRequest = downloadFileRequest;
this.objectOperation = objectOperation;
this.progressListener = progressListener;
this.simplifiedMeta = simplifiedMeta;
}

@Override
Expand Down Expand Up @@ -643,6 +647,8 @@ public PartResult call() throws Exception {
objectMetadata = ossObj.getObjectMetadata();
content = ossObj.getObjectContent();

this.verifyObjectMeta(ossObj, getObjectRequest, simplifiedMeta);

byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
int bytesRead = 0;
while ((bytesRead = IOUtils.readNBytes(content, buffer, 0, buffer.length)) > 0) {
Expand Down Expand Up @@ -680,6 +686,29 @@ public PartResult call() throws Exception {
return tr;
}

public void verifyObjectMeta(OSSObject ossObj, GetObjectRequest getObjectRequest, SimplifiedObjectMeta simplifiedMeta) {
if(ossObj.getResponse().getStatusCode() == 206){
long rangeSetStart = getObjectRequest.getRange()[0];

String returnSet = ossObj.getResponse().getHeaders().get("Content-Range");
String returnStartStr = returnSet.substring(returnSet.indexOf("bytes")+6,(returnSet.indexOf("-")));
Long returnSetStart = -1L;
try {
returnSetStart = Long.parseLong(returnStartStr);
} catch(Exception e){
logException("Failed to convert string to long", e);
}

if(rangeSetStart != returnSetStart){
throw new ClientException("Range get fail, expect offset:" + getObjectRequest.getRange() + "return offset:" + returnSet);
}
}

if(simplifiedMeta.getLastModified().hashCode() != ossObj.getObjectMetadata().getLastModified().hashCode()){
throw new ClientException("The last modification time of the file has changed, Last file modification time:" + ossObj.getObjectMetadata().getLastModified() + ", The file modification time:" + simplifiedMeta.getLastModified());
}
}

public ObjectMetadata GetobjectMetadata() {
return objectMetadata;
}
Expand All @@ -692,6 +721,7 @@ public ObjectMetadata GetobjectMetadata() {
private OSSObjectOperation objectOperation;
private ObjectMetadata objectMetadata;
private ProgressListener progressListener;
private SimplifiedObjectMeta simplifiedMeta;
}

private ArrayList<DownloadPart> splitFile(long start, long objectSize, long partSize) {
Expand Down
28 changes: 14 additions & 14 deletions src/main/java/com/aliyun/oss/internal/OSSUploadOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,7 @@
import com.aliyun.oss.event.ProgressEventType;
import com.aliyun.oss.event.ProgressListener;
import com.aliyun.oss.event.ProgressPublisher;
import com.aliyun.oss.model.CannedAccessControlList;
import com.aliyun.oss.model.CompleteMultipartUploadRequest;
import com.aliyun.oss.model.CompleteMultipartUploadResult;
import com.aliyun.oss.model.InitiateMultipartUploadRequest;
import com.aliyun.oss.model.InitiateMultipartUploadResult;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.PartETag;
import com.aliyun.oss.model.Payer;
import com.aliyun.oss.model.UploadFileRequest;
import com.aliyun.oss.model.UploadFileResult;
import com.aliyun.oss.model.UploadPartRequest;
import com.aliyun.oss.model.UploadPartResult;
import com.aliyun.oss.model.*;

/**
* OSSUploadOperation
Expand Down Expand Up @@ -477,11 +466,12 @@ private ArrayList<PartResult> upload(UploadCheckPoint uploadCheckPoint, UploadFi
ProgressPublisher.publishRequestBytesTransferred(listener, completedLength);
uploadFileRequest.setProgressListener(null);

File file = new File(uploadCheckPoint.uploadFile);
// Upload parts.
for (int i = 0; i < uploadCheckPoint.uploadParts.size(); i++) {
if (!uploadCheckPoint.uploadParts.get(i).isCompleted) {
futures.add(service.submit(new Task(i, "upload-" + i, uploadCheckPoint, i, uploadFileRequest,
multipartOperation, listener)));
multipartOperation, listener, file.lastModified())));
} else {
taskResults.add(new PartResult(i + 1, uploadCheckPoint.uploadParts.get(i).offset,
uploadCheckPoint.uploadParts.get(i).size, uploadCheckPoint.uploadParts.get(i).crc));
Expand Down Expand Up @@ -517,21 +507,24 @@ class Task implements Callable<PartResult> {

public Task(int id, String name, UploadCheckPoint uploadCheckPoint, int partIndex,
UploadFileRequest uploadFileRequest, OSSMultipartOperation multipartOperation,
ProgressListener progressListener) {
ProgressListener progressListener, long lastModified) {
this.id = id;
this.name = name;
this.uploadCheckPoint = uploadCheckPoint;
this.partIndex = partIndex;
this.uploadFileRequest = uploadFileRequest;
this.multipartOperation = multipartOperation;
this.progressListener = progressListener;
this.lastModified = lastModified;
}

@Override
public PartResult call() throws Exception {
PartResult tr = null;
InputStream instream = null;

File file = new File(uploadCheckPoint.uploadFile);
this.verifyObjectMeta(lastModified, file.lastModified());
try {
UploadPart uploadPart = uploadCheckPoint.uploadParts.get(partIndex);
tr = new PartResult(partIndex + 1, uploadPart.offset, uploadPart.size);
Expand Down Expand Up @@ -584,13 +577,20 @@ public PartResult call() throws Exception {
return tr;
}

public void verifyObjectMeta(long lastModified, long currentLastModified) {
if(lastModified != currentLastModified){
throw new ClientException("The last modification time of the file has changed, Last file modification time:" + currentLastModified + ", The file modification time:" + lastModified);
}
}

private int id;
private String name;
private UploadCheckPoint uploadCheckPoint;
private int partIndex;
private UploadFileRequest uploadFileRequest;
private OSSMultipartOperation multipartOperation;
private ProgressListener progressListener;
private long lastModified;
}

private CompleteMultipartUploadResult complete(UploadCheckPoint uploadCheckPoint,
Expand Down