Amazon aws s3上傳文件，並設置爲公共可讀
直接上硬菜：
1.依賴
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.625</version>
</dependency>
2.編碼
public static String uploadToS3(MultipartFile file) throws IOException {
String perfix = "https://xxxxx.s3-us-west-1.amazonaws.com/";
String bucketName = "txxxxx";
if (file.isEmpty()) {
return "上傳文件不能爲空";
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(file.getContentType());
metadata.setContentLength(file.getSize());
String key = UUID.randomUUID().toString().replaceAll("-", "") + "." + getFileType(file.getOriginalFilename());
String fileTypeByContentType = getFileTypeByContentType(file.getContentType());
if ("image".equals(fileTypeByContentType)) {
bucketName = "xxx-img";
perfix = "https://xxx-img.s3-us-west-1.amazonaws.com/";
} else if ("video".equals(fileTypeByContentType)) {
bucketName = "xxx-video1";
perfix = "https://xxx-video1.s3-us-west-1.amazonaws.com/";
} else {
bucketName = "xxxx-other";
perfix = "https://xxx-other.s3-us-west-1.amazonaws.com/";
}
try {
//驗證名稱爲bucketName的bucket是否存在，不存在則創建
if (!checkBucketExists(s3Client, bucketName)) {
s3Client.createBucket(bucketName);
}
zhi
/*之前被誤導，一直上傳上此的文件，返回一個鏈接帶有有效期，而且最七天，各種想辦法，其實是寫法就錯誤了，
應該用下面的這種寫法withCannedAcl，設置ACL權限就好，希望大家避坑*/
//開始上傳文件
s3Client.putObject(new PutObjectRequest(bucketName, key, file.getInputStream(), metadata)
.withCannedAcl(CannedAccessControlList.PublicRead));
String url = perfix + key;
if (url == null) {
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), " can't get s3 file url!");
}
return url.toString();
} catch (AmazonServiceException ase) {
ase.printStackTrace();
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Error Message: " + ase.getMessage());
log.info("HTTP Status Code: " + ase.getStatusCode());
log.info("AWS Error Code: " + ase.getErrorCode());
log.info("Error Type: " + ase.getErrorType());
log.info("Request ID: " + ase.getRequestId());
log.info(ase.getMessage(), ase);
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} catch (AmazonClientException ace) {
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
log.info("Error Message: " + ace.getMessage());
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} finally {
}
}
就是這麼硬
在給定bucket新建文件夾
其實非常檢查，簡單，只需要在key設置的時候，加上文件夾名，再加個斜杆/即可
核心代碼在這裏，
記得上傳改爲用線程池，多線程去異步執行就好，如果需要返回結果，則調用callable方法實現，把結果返回給主線程，
以實現線程間通訊
今天想記錄一下工作中一個場景：
上傳一個zip包（是網站的一個主題活動頁面，裏面是隻能有一個html文件和多個css/js/image等之類的資源文件）
後臺服務器解壓，並上上傳到asw的s3存儲。
解決步驟
1.upload文件到服務器本地
2.結果zip文件
3.用多線程 實現批量上傳文件到s3，並把上傳返回的結果記錄到內存map中，用以在第4步中替換
4.本來想着服務端去I/O html文件，readline,然後根據step3中的map，遍歷該map再替換，發現其實挺慢
5.於是做了決定，讓前端自己去修改html中資源路徑，直接告訴她替換規則爲http://xxxxx/壓縮文件名字/資源名
這樣就帶來的好處
1.可以直接異步去上傳這個一個html文件，立刻返回個前端結果
2.步驟3中不需要存每一個上傳的返回路徑了，他可以完全異步去做，
3.步驟4中也不要去I/O html文件，readline，再去遍歷了
好處大大的啊！就這麼愉快的決定了
代碼如下
public static String generatorPage(File zipFile) {
try {
String zipFileOriginalFilename = zipFile.getName();
String zipfileSuffix = FileUtil.getFileType(zipFileOriginalFilename);
if (!zipfileSuffix.equals("zip")){
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_ONLY_ZIP);
}
//服務器保存路徑-------配置文件讀取
String serverFilePath = PropertyUtils.getftpProp("upload_zipfile_path");
/**
* step1.上傳文件到應用服務器
*/
byte[] zipFileByteArrays = getFileByteArray(zipFile);
FileUtil.uploadFile(zipFileByteArrays, serverFilePath, zipFileOriginalFilename);
String uploadFilePath = serverFilePath + zipFileOriginalFilename;
/**
* step2.將該文件解壓
*/
ZipUtil.unZipFiles(uploadFilePath, serverFilePath);
String unzipFilePath = uploadFilePath.substring(0, uploadFilePath.lastIndexOf("."));
File file = new File(unzipFilePath);
String pageName = file.getName();
List<File> allUnZipFiles = FileUtil.getAllFiles(unzipFilePath);
File htmlFile = null;
/**
* step3.上傳解壓後的html文件到s3
*/
// Map replaceMap = new HashMap();
int htmlfileNumber = 0;
for (File unzipFile : allUnZipFiles) {
String unzipfileName = unzipFile.getName();
String unzipfileSuffix1 = FileUtil.getFileType(unzipfileName);
if (unzipfileSuffix1.equals("html") || unzipfileSuffix1.equals("htm")) {//如果是html文件
htmlFile = unzipFile;
htmlfileNumber++;
}
}
if (htmlfileNumber == 0 || htmlfileNumber > 1) {
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_HTML_NUMBER);
}
long start = System.currentTimeMillis();
String s3Path = uploadS3ThreadPoolAsync(pageName, htmlFile).get();
long end1 = System.currentTimeMillis();
System.out.println("上傳hmtl頁面完畢，耗時：" + (end1 - start));
/**
* step4.上傳非html文件到s3
*/
for (File unzipFile : allUnZipFiles) {
String unzipfileName = unzipFile.getName();
String unzipfileSuffix = FileUtil.getFileType(unzipfileName);
if (!"zip".equals(unzipfileSuffix) && !"DS_Store".equals(unzipfileSuffix) && !"html".equals(unzipfileSuffix) && !"htm".equals(unzipfileSuffix)) {
uploadS3ThreadPoolAsync(pageName, unzipFile);
}
}
//step4.替換html文件中資源路徑[這一步省去，讓前端同學處理，把所有資源類的文件路徑都替換爲：
// https://xxxxx.amazonaws.com/壓縮包文件名/14769441969842358.png]
//FileUtil.alterStringToCreateNewFile(htmlFile.getAbsolutePath(), replaceMap);
long end2 = System.currentTimeMillis();
System.out.println(" ----多線程，異步發佈任務結束(後臺異步去運行)-----：" + (end2 - end1));
System.out.println("主線程運行結束，耗時：" + (end2 - start));
return s3Path;
} catch (Exception e) {
e.printStackTrace();
throw new BizException(GlobalExceptionEnum.UPLOAD_GENERATOR_PAGE_FAIL);
}
}
private static Future<String> uploadS3ThreadPoolAsync(String pageName, File htmlFile) throws InterruptedException, ExecutionException {
Future<String> submit = executorService.submit(() -> {
try {
String htmlS3Path = AmazonS3Util.uploadToS3Async(htmlFile, false, pageName);
return htmlS3Path;
} catch (IOException e) {
e.printStackTrace();
return "";
}
});
return submit;
}
/**
*
* @param file 要上傳的文件,文件爲普通文件
* @param issinglefile 是否是單個文件上傳
* @param pageName 活動頁壓縮名稱
* @return
* @throws IOException
*/
public static String uploadToS3Async(File file, boolean issinglefile, String pageName) throws IOException {
if (file.length() == 0) {
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_IS_EMPTY);
}
long size = file.length();
String path = file.getPath();
String originalFilename = file.getName();
InputStream inputStream = new FileInputStream(file);
String contentType = FileUtil.getFileContentTypeByPath(path);
return getUploadS3Path(contentType, originalFilename, size, inputStream, issinglefile, pageName);
}
/**
*
* @param multipartFile 要上傳的文件，文件爲Multipart文件
* @param issinglefile 是否是單個文件上傳
* @param pageName 活動頁壓縮名稱
* @return
* @throws IOException
*/
public static String uploadToS3Async(MultipartFile multipartFile, boolean issinglefile, String pageName) throws IOException {
if (multipartFile.isEmpty()) {
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_IS_EMPTY);
}
String contentType = multipartFile.getContentType();
String originalFilename = multipartFile.getOriginalFilename();
long size = multipartFile.getSize();
InputStream inputStream = multipartFile.getInputStream();
return getUploadS3Path(contentType, originalFilename, size, inputStream, issinglefile, pageName);
}
/**
* 普通單個文件上傳
*
* @param contentType
* @param originalFilename
* @param size
* @param inputStream
* @return
*/
private static String getUploadS3Path(String contentType, String originalFilename, long size, InputStream inputStream, boolean issinglefile, String pageName) {
if (!issinglefile) {
bucketName = "tfc-page";
perfix = "https://tfc-page.s3-us-west-1.amazonaws.com/";
} else {
String fileTypeByContentType = getFileTypeByContentType(contentType);
if ("image".equals(fileTypeByContentType)) {
bucketName = "tfc-img";
perfix = "https://tfc-img.s3-us-west-1.amazonaws.com/";
} else if ("video".equals(fileTypeByContentType)) {
bucketName = "tfc-video1";
perfix = "https://tfc-video1.s3-us-west-1.amazonaws.com/";
} else {
bucketName = "tfc-other";
perfix = "https://tfc-other.s3-us-west-1.amazonaws.com/";
}
}
return uploadToS3Asyn(contentType, originalFilename, size, inputStream, pageName);
}
private static String uploadToS3Asyn(String contentType, String originalFilename, long size, InputStream inputStream, String pageName) {
Future<String> submit = executorService.submit(() -> {
String key = "";
try {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(contentType);
metadata.setContentLength(size);
if (pageName == null || "".equals(pageName)) {
key = DateUtil.getDays()+"/"+UUID.randomUUID().toString().replaceAll("-", "") + "." + getFileType(originalFilename);
} else {
key = pageName + "/" + originalFilename;
}
//驗證名稱爲bucketName的bucket是否存在，不存在則創建
if (!checkBucketExists(s3Client, bucketName)) {
s3Client.createBucket(bucketName);
}
System.out.format("線程%s 正在上傳 %s",Thread.currentThread().getName(),key);
System.out.println();
//開始上傳文件
s3Client.putObject(new PutObjectRequest(bucketName, key, inputStream, metadata)
.withCannedAcl(CannedAccessControlList.PublicRead));
System.out.format("線程%s 上傳完畢",Thread.currentThread().getName());
System.out.println();
String url = perfix + key;
return url;
} catch (AmazonServiceException ase) {
System.out.format("-----__------------上傳key=%s時異常",key);
ase.printStackTrace();
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Error Message: " + ase.getMessage());
log.info("HTTP Status Code: " + ase.getStatusCode());
log.info("AWS Error Code: " + ase.getErrorCode());
log.info("Error Type: " + ase.getErrorType());
log.info("Request ID: " + ase.getRequestId());
log.info(ase.getMessage(), ase);
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} catch (AmazonClientException ace) {
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
log.info("Error Message: " + ace.getMessage());
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} finally {
}
});
String s3Path = null;
try {
s3Path = submit.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
return s3Path;
}