Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Download all fix #478

Merged
merged 3 commits into from
Apr 10, 2019
Merged

Download all fix #478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a4738e4
Fix download all
coderReview Apr 10, 2019
0025935
Restore dev files
coderReview Apr 10, 2019
367003a
Restore dev files
coderReview Apr 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion build-dependencies.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,9 @@
<property name="asm-5.2.jar" value="${ext_libdir}/asm/asm-5.2.jar"/>
<property name="asm-commons-5.2.jar" value="${ext_libdir}/asm/asm-commons-5.2.jar"/>
<property name="asm-tree-5.2.jar" value="${ext_libdir}/asm/asm-tree-5.2.jar"/>
<property name="aws-java-sdk.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-1.0.004.jar"/>
<property name="aws-java-sdk.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-1.11.490.jar"/>
<property name="aws-java-sdk-core.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-core-1.11.490.jar"/>
<property name="aws-java-sdk-s3.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar"/>
<property name="jackson-core.jar" value="${ext_libdir}/jackson/2.8.1/jackson-core-asl.jar"/>
<property name="jackson-mapper.jar" value="${ext_libdir}/jackson/2.8.1/jackson-mapper-asl.jar"/>
<property name="jackson-annotations-2.8.1.jar" value="${ext_libdir}/jackson/2.8.1/jackson-annotations-2.8.1.jar"/>
Expand Down Expand Up @@ -635,6 +637,8 @@
<pathelement location="${encoder.jar}"/>
<pathelement location="${javaee.jar}"/>
<pathelement location="${aws-java-sdk.jar}"/>
<pathelement location="${aws-java-sdk-core.jar}"/>
<pathelement location="${aws-java-sdk-s3.jar}"/>
<pathelement location="${jrss.jar}"/>
<pathelement location="${joda-time.jar}"/>
<pathelement location="${mime-util.jar}"/>
Expand Down
2 changes: 2 additions & 0 deletions build.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,8 @@
<copy file="${asm-commons-5.2.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${asm-tree-5.2.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${aws-java-sdk.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${aws-java-sdk-core.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${aws-java-sdk-s3.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${axis.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${commons-dbcp.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
<copy file="${commons-discovery.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
Expand Down
Binary file added lib/third_party/aws-java-sdk/aws-java-sdk-1.11.490.jar
View file
Open in desktop
Binary file not shown.
Binary file added lib/third_party/aws-java-sdk/aws-java-sdk-core-1.11.490.jar
View file
Open in desktop
Binary file not shown.
Binary file added lib/third_party/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar
View file
Open in desktop
Binary file not shown.
64 changes: 51 additions & 13 deletions ...om/topcoder/direct/services/view/action/contest/DownloadAllSoftwareSubmissionsAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.AmazonS3URI;
import com.topcoder.direct.services.view.action.contest.launch.ContestAction;
import com.topcoder.direct.services.view.dto.contest.ContestRoundType;
import com.topcoder.direct.services.view.dto.contest.ContestType;
Expand All @@ -15,18 +16,19 @@
import com.topcoder.service.project.SoftwareCompetition;
import com.topcoder.servlet.request.FileUpload;
import com.topcoder.servlet.request.UploadedFile;
import com.topcoder.shared.util.logging.Logger;
import org.apache.commons.io.FilenameUtils;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

/**
Expand Down Expand Up @@ -65,6 +67,11 @@
*/
public class DownloadAllSoftwareSubmissionsAction extends ContestAction {

/**
* Logging instance
*/
private static final Logger logger = Logger.getLogger(DownloadAllSoftwareSubmissionsAction.class);

/**
* The id of the final submission type.
*
Expand Down Expand Up @@ -287,15 +294,42 @@ public void run() {
byte[] buffer = new byte[8192];
int read;
InputStream is = null;
DefaultHttpClient httpClient = new DefaultHttpClient();
try {
for (Submission sub : submissionsToDownload) {
String submissionFileZipName;
// url != null is s3
// url != null is s3/external url
if (sub.getUpload().getUrl() != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(sub.getUpload().getUrl())));
is = s3Object.getObjectContent();
submissionFileZipName = DirectUtils.getS3FileKey(sub.getUpload().getUrl());
try {
AmazonS3URI s3Uri = DirectUtils.getS3Uri(sub.getUpload().getUrl());
if (s3Uri != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(sub.getUpload().getUrl())));
is = s3Object.getObjectContent();
submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getS3FileKey(sub.getUpload().getUrl());
} else {
// external url other than s3
HttpGet request = new HttpGet(sub.getUpload().getUrl());
HttpResponse response = httpClient.execute(request);
// skip status code >=400
if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external");
}

HttpEntity entity = response.getEntity();
if (entity != null) {
is = entity.getContent();
} else {
throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external");
}
submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getFileNameFromUrl(sub.getUpload().getUrl());
}
} catch (Exception e) {
logger.error("Fail to get submission " + sub.getId() + " url: " + sub.getUpload().getUrl() +
" message: " + e.getMessage());
logger.info("Skipping submission " + sub.getId() + " url: " + sub.getUpload().getUrl());
continue;
}
} else {
UploadedFile file;
if (DirectUtils.isStudio(contest)) {
Expand Down Expand Up @@ -350,6 +384,10 @@ public void run() {
// ignore
}
}
} finally {
if (httpClient != null) {
httpClient.getConnectionManager().shutdown();
}
}
try {
zos.close();
Expand Down
47 changes: 38 additions & 9 deletions ...in/com/topcoder/direct/services/view/action/contest/DownloadSoftwareSubmissionAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.AmazonS3URI;
import com.topcoder.direct.services.view.action.BaseDirectStrutsAction;
import com.topcoder.direct.services.view.dto.contest.ContestType;
import com.topcoder.direct.services.view.util.DirectUtils;
Expand All @@ -13,6 +15,12 @@
import com.topcoder.service.project.SoftwareCompetition;
import com.topcoder.servlet.request.FileUpload;
import com.topcoder.servlet.request.UploadedFile;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import java.io.InputStream;

Expand Down Expand Up @@ -91,9 +99,9 @@ public class DownloadSoftwareSubmissionAction extends BaseDirectStrutsAction {
private SoftwareCompetition contest;

/**
* S3 url of uploaded file. Null if it use local file
* External url of uploaded file. Null if it use local file
*/
private String s3Url;
private String externalUrl;

/**
* S3 bucket
Expand Down Expand Up @@ -144,7 +152,7 @@ protected void executeAction() throws Exception {
uploadedFile = fileUpload.getUploadedFile(submission.getUpload().getParameter());
}
} else {
s3Url = submission.getUpload().getUrl();
externalUrl = submission.getUpload().getUrl();
}

}
Expand All @@ -157,10 +165,27 @@ protected void executeAction() throws Exception {
* if any error occurs when getting the input stream of the uploaded file.
*/
public InputStream getInputStream() throws Exception {
if (s3Url != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(s3Url)));
return s3Object.getObjectContent();
if (externalUrl != null) {
AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl);
if (s3Uri != null) {
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
DirectUtils.getS3FileKey(externalUrl)));
return s3Object.getObjectContent();
} else {
DefaultHttpClient httpClient = new DefaultHttpClient();
HttpGet request = new HttpGet(externalUrl);
HttpResponse response = httpClient.execute(request);
// skip status code >=400
if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external");
}

HttpEntity entity = response.getEntity();
if (entity == null) {
throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external");
}
return entity.getContent();
}
}

if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) {
Expand Down Expand Up @@ -188,8 +213,12 @@ public InputStream getInputStream() throws Exception {
* if any error occurs when getting the file name of the uploaded file.
*/
public String getContentDisposition() throws Exception {
if (s3Url != null) {
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(s3Url) + "\"";
if (externalUrl != null) {
AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl);
if (s3Uri != null) {
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(externalUrl) + "\"";
}
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getFileNameFromUrl(externalUrl) + "\"";
}

if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) {
Expand Down
4 changes: 2 additions & 2 deletions src/java/main/com/topcoder/direct/services/view/ajax/SoftwareCompetitionBeanProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,11 +214,11 @@ private Object getMapResult(SoftwareCompetition bean) {

// retrieve review scorecard id.
for(com.topcoder.project.phases.Phase phase : bean.getProjectPhases().getAllPhases()){
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName())){
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){
result.put("reviewScorecardId", phase.getAttributes().get("Scorecard ID").toString());
}

if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName())){
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){
result.put("iterativeReviewScorecardId", phase.getAttributes().get("Scorecard ID").toString());
}
}
Expand Down
36 changes: 34 additions & 2 deletions src/java/main/com/topcoder/direct/services/view/util/DirectUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3URI;
import com.opensymphony.xwork2.ActionContext;
import com.topcoder.clients.dao.ProjectContestFeePercentageService;
import com.topcoder.clients.dao.ProjectContestFeeService;
Expand Down Expand Up @@ -4075,9 +4076,40 @@ public static AmazonS3Client getS3Client() {
* @throws Exception if any exceptions occurs
*/
public static String getS3FileKey(String url) throws Exception {
AmazonS3URI s3Uri = getS3Uri(url);
if (s3Uri == null) {
return null;
}
return s3Uri.getKey();
}

/**
* Get upload uri from url.
*
* @param url upload url
* @return s3 uri
*/
public static AmazonS3URI getS3Uri(String url) {
try {
AmazonS3URI s3Uri = new AmazonS3URI(url);
return s3Uri;
} catch (IllegalArgumentException ex) {
// url doesn't seem to be a valid
return null;
}
}

/**
* Get filename from URL
*
* @param url
* @return filename
* @throws Exception
*/
public static String getFileNameFromUrl(String url) throws Exception {
String path = new URL(url).getPath();
int sep = path.lastIndexOf( '/' );
return ( sep < 0 ) ? path : path.substring( sep + 1 );
int sep = path.lastIndexOf('/');
return (sep < 0) ? path : path.substring(sep + 1);
}

/**
Expand Down