Skip to content

Commit

Permalink
Merge pull request #478 from appirio-tech/download-all-fix-2
Browse files Browse the repository at this point in the history 
Download all fix
  • Loading branch information
@coderReview
coderReview authored Apr 10, 2019
2 parents 712a277 + 367003a commit 2699887
Show file tree
Hide file tree
Showing 9 changed files with 132 additions and 27 deletions.
6 changes: 5 additions & 1 deletion build-dependencies.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,9 @@
<property name="asm-5.2.jar" value="${ext_libdir}/asm/asm-5.2.jar"/>
<property name="asm-commons-5.2.jar" value="${ext_libdir}/asm/asm-commons-5.2.jar"/>
<property name="asm-tree-5.2.jar" value="${ext_libdir}/asm/asm-tree-5.2.jar"/>
<property name="aws-java-sdk.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-1.0.004.jar"/>
<property name="aws-java-sdk.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-1.11.490.jar"/>
<property name="aws-java-sdk-core.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-core-1.11.490.jar"/>
<property name="aws-java-sdk-s3.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar"/>
<property name="jackson-core.jar" value="${ext_libdir}/jackson/2.8.1/jackson-core-asl.jar"/>
<property name="jackson-mapper.jar" value="${ext_libdir}/jackson/2.8.1/jackson-mapper-asl.jar"/>
<property name="jackson-annotations-2.8.1.jar" value="${ext_libdir}/jackson/2.8.1/jackson-annotations-2.8.1.jar"/>
Expand Down Expand Up @@ -635,6 +637,8 @@
<pathelement location="${encoder.jar}"/>
<pathelement location="${javaee.jar}"/>
<pathelement location="${aws-java-sdk.jar}"/>
<pathelement location="${aws-java-sdk-core.jar}"/>
<pathelement location="${aws-java-sdk-s3.jar}"/>
<pathelement location="${jrss.jar}"/>
<pathelement location="${joda-time.jar}"/>
<pathelement location="${mime-util.jar}"/>
Expand Down
2 changes: 2 additions & 0 deletions build.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,8 @@
<copy file="${asm-commons-5.2.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${asm-tree-5.2.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${aws-java-sdk.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${aws-java-sdk-core.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${aws-java-sdk-s3.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${axis.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${commons-dbcp.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${commons-discovery.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
Expand Down
Binary file added lib/third_party/aws-java-sdk/aws-java-sdk-1.11.490.jar
View file
Binary file not shown.
Binary file added lib/third_party/aws-java-sdk/aws-java-sdk-core-1.11.490.jar
View file
Binary file not shown.
Binary file added lib/third_party/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar
View file
Binary file not shown.
64 changes: 51 additions & 13 deletions ...om/topcoder/direct/services/view/action/contest/DownloadAllSoftwareSubmissionsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.AmazonS3URI;
import com.topcoder.direct.services.view.action.contest.launch.ContestAction;
import com.topcoder.direct.services.view.dto.contest.ContestRoundType;
import com.topcoder.direct.services.view.dto.contest.ContestType;
Expand All @@ -15,18 +16,19 @@
import com.topcoder.service.project.SoftwareCompetition;
import com.topcoder.servlet.request.FileUpload;
import com.topcoder.servlet.request.UploadedFile;
import com.topcoder.shared.util.logging.Logger;
import org.apache.commons.io.FilenameUtils;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

/**
Expand Down Expand Up @@ -65,6 +67,11 @@
*/
public class DownloadAllSoftwareSubmissionsAction extends ContestAction {

/**
* Logging instance
*/
private static final Logger logger = Logger.getLogger(DownloadAllSoftwareSubmissionsAction.class);

/**
* The id of the final submission type.
*
Expand Down Expand Up @@ -287,15 +294,42 @@ public void run() {
byte[] buffer = new byte[8192];
int read;
InputStream is = null;
DefaultHttpClient httpClient = new DefaultHttpClient();
try {
for (Submission sub : submissionsToDownload) {
String submissionFileZipName;
// url != null is s3
// url != null is s3/external url
if (sub.getUpload().getUrl() != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(sub.getUpload().getUrl())));
is = s3Object.getObjectContent();
submissionFileZipName = DirectUtils.getS3FileKey(sub.getUpload().getUrl());
try {
AmazonS3URI s3Uri = DirectUtils.getS3Uri(sub.getUpload().getUrl());
if (s3Uri != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(sub.getUpload().getUrl())));
is = s3Object.getObjectContent();
submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getS3FileKey(sub.getUpload().getUrl());
} else {
// external url other than s3
HttpGet request = new HttpGet(sub.getUpload().getUrl());
HttpResponse response = httpClient.execute(request);
// skip status code >=400
if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external");
}

HttpEntity entity = response.getEntity();
if (entity != null) {
is = entity.getContent();
} else {
throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external");
}
submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getFileNameFromUrl(sub.getUpload().getUrl());
}
} catch (Exception e) {
logger.error("Fail to get submission " + sub.getId() + " url: " + sub.getUpload().getUrl() +
" message: " + e.getMessage());
logger.info("Skipping submission " + sub.getId() + " url: " + sub.getUpload().getUrl());
continue;
}
} else {
UploadedFile file;
if (DirectUtils.isStudio(contest)) {
Expand Down Expand Up @@ -350,6 +384,10 @@ public void run() {
// ignore
}
}
} finally {
if (httpClient != null) {
httpClient.getConnectionManager().shutdown();
}
}
try {
zos.close();
Expand Down
47 changes: 38 additions & 9 deletions ...in/com/topcoder/direct/services/view/action/contest/DownloadSoftwareSubmissionAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.AmazonS3URI;
import com.topcoder.direct.services.view.action.BaseDirectStrutsAction;
import com.topcoder.direct.services.view.dto.contest.ContestType;
import com.topcoder.direct.services.view.util.DirectUtils;
Expand All @@ -13,6 +15,12 @@
import com.topcoder.service.project.SoftwareCompetition;
import com.topcoder.servlet.request.FileUpload;
import com.topcoder.servlet.request.UploadedFile;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import java.io.InputStream;

Expand Down Expand Up @@ -91,9 +99,9 @@ public class DownloadSoftwareSubmissionAction extends BaseDirectStrutsAction {
private SoftwareCompetition contest;

/**
* S3 url of uploaded file. Null if it use local file
* External url of uploaded file. Null if it use local file
*/
private String s3Url;
private String externalUrl;

/**
* S3 bucket
Expand Down Expand Up @@ -144,7 +152,7 @@ protected void executeAction() throws Exception {
uploadedFile = fileUpload.getUploadedFile(submission.getUpload().getParameter());
}
} else {
s3Url = submission.getUpload().getUrl();
externalUrl = submission.getUpload().getUrl();
}

}
Expand All @@ -157,10 +165,27 @@ protected void executeAction() throws Exception {
* if any error occurs when getting the input stream of the uploaded file.
*/
public InputStream getInputStream() throws Exception {
if (s3Url != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(s3Url)));
return s3Object.getObjectContent();
if (externalUrl != null) {
AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl);
if (s3Uri != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(externalUrl)));
return s3Object.getObjectContent();
} else {
DefaultHttpClient httpClient = new DefaultHttpClient();
HttpGet request = new HttpGet(externalUrl);
HttpResponse response = httpClient.execute(request);
// skip status code >=400
if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external");
}

HttpEntity entity = response.getEntity();
if (entity == null) {
throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external");
}
return entity.getContent();
}
}

if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) {
Expand Down Expand Up @@ -188,8 +213,12 @@ public InputStream getInputStream() throws Exception {
* if any error occurs when getting the file name of the uploaded file.
*/
public String getContentDisposition() throws Exception {
if (s3Url != null) {
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(s3Url) + "\"";
if (externalUrl != null) {
AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl);
if (s3Uri != null) {
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(externalUrl) + "\"";
}
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getFileNameFromUrl(externalUrl) + "\"";
}

if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) {
Expand Down
4 changes: 2 additions & 2 deletions src/java/main/com/topcoder/direct/services/view/ajax/SoftwareCompetitionBeanProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,11 +214,11 @@ private Object getMapResult(SoftwareCompetition bean) {

// retrieve review scorecard id.
for(com.topcoder.project.phases.Phase phase : bean.getProjectPhases().getAllPhases()){
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName())){
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){
result.put("reviewScorecardId", phase.getAttributes().get("Scorecard ID").toString());
}

if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName())){
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){
result.put("iterativeReviewScorecardId", phase.getAttributes().get("Scorecard ID").toString());
}
}
Expand Down
36 changes: 34 additions & 2 deletions src/java/main/com/topcoder/direct/services/view/util/DirectUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3URI;
import com.opensymphony.xwork2.ActionContext;
import com.topcoder.clients.dao.ProjectContestFeePercentageService;
import com.topcoder.clients.dao.ProjectContestFeeService;
Expand Down Expand Up @@ -4075,9 +4076,40 @@ public static AmazonS3Client getS3Client() {
* @throws Exception if any exceptions occurs
*/
public static String getS3FileKey(String url) throws Exception {
AmazonS3URI s3Uri = getS3Uri(url);
if (s3Uri == null) {
return null;
}
return s3Uri.getKey();
}

/**
* Get upload uri from url.
*
* @param url upload url
* @return s3 uri
*/
public static AmazonS3URI getS3Uri(String url) {
try {
AmazonS3URI s3Uri = new AmazonS3URI(url);
return s3Uri;
} catch (IllegalArgumentException ex) {
// url doesn't seem to be a valid
return null;
}
}

/**
* Get filename from URL
*
* @param url
* @return filename
* @throws Exception
*/
public static String getFileNameFromUrl(String url) throws Exception {
String path = new URL(url).getPath();
int sep = path.lastIndexOf( '/' );
return ( sep < 0 ) ? path : path.substring( sep + 1 );
int sep = path.lastIndexOf('/');
return (sep < 0) ? path : path.substring(sep + 1);
}

/**
Expand Down

0 comments on commit 2699887

Please sign in to comment.