Amazon S3介绍及实例util
2017-09-27 16:14
204 查看
Amazon S3 功能介绍
一 .Amazon S3介绍Amazon Simple Storage Service (Amazon S3) 是一种对象存储，它具有简单的 Web 服务接口，可用于在 Web 上的任何位置存储和检索任意数量的数据。它能够提供 99.999999999% 的持久性，并且可以在全球大规模传递数万亿对象。
客户使用 S3 作为云原生应用程序的主要存储；作为分析的批量存储库或“数据湖”；作为备份和恢复以及灾难恢复的目标；并将其与无服务器计算配合使用。
使用 Amazon 的云数据迁移选项，客户可以轻松地将大量数据移入或移出 Amazon S3。数据在存储到 S3 中之后，会自动采用成本更低、存储期限更长的云存储类
(如 S3 Standard – Infrequent Access 和 Amazon Glacier) 进行存档。
二.Java S3 Example
准备工作：
1.导入依赖包
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.9.2</version>
</dependency>
2.在s3服务中创建用户，获取用户的Access key和Secret Access Key，使用这个作为凭证连接s3
3.在s3服务中配置AWSConnector和AmazonS3FullAccess的连接权限。可以通过这个配置，在访问的时候进行authenticate验证。
s3 api简单操作：
1.创建凭证
AWSCredentials credentials = new BasicAWSCredentials("YourAccessKeyID", "YourSecretAccessKey");
2.创建S3 Client
AmazonS3 s3client = new AmazonS3Client(credentials);
3.创建Bucket
String bucketName = "javatutorial-net-example-bucket";
s3client.createBucket(bucketName);
4.获取s3 Bucket的list
for (Bucket bucket : s3client.listBuckets()) {
System.out.println(" - " + bucket.getName());
}
5.在s3 Bucket中创建文件
public static void createFolder(String bucketName, String folderName, AmazonS3 client) {
// create meta-data for your folder and set content-length to 0
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(0);
// create empty content
InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
// create a PutObjectRequest passing the folder name suffixed by /
PutObjectRequest putObjectRequest = new PutObje
13452
ctRequest(bucketName,
folderName + SUFFIX, emptyContent, metadata);
// send request to S3 to create folder
client.putObject(putObjectRequest);
}
6.上传文件
String fileName = folderName + SUFFIX + "testvideo.mp4";
s3client.putObject(new PutObjectRequest(bucketName, fileName,
new File("C:\\Users\\user\\Desktop\\testvideo.mp4")));
7.删除Bucket
s3client.deleteBucket(bucketName);
8.删除文件
s3client.deleteObject(bucketName, fileName);
完整实例
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectSummary;
public class AmazonS3Example {
private static final String SUFFIX = "/";
public static void main(String[] args) {
// credentials object identifying user for authentication
// user must have AWSConnector and AmazonS3FullAccess for
// this example to work
AWSCredentials credentials = new BasicAWSCredentials(
"YourAccessKeyID",
"YourSecretAccessKey");
// create a client connection based on credentials
AmazonS3 s3client = new AmazonS3Client(credentials);
// create bucket - name must be unique for all S3 users
String bucketName = "javatutorial-net-example-bucket";
s3client.createBucket(bucketName);
// list buckets
for (Bucket bucket : s3client.listBuckets()) {
System.out.println(" - " + bucket.getName());
}
// create folder into bucket
String folderName = "testfolder";
createFolder(bucketName, folderName, s3client);
// upload file to folder and set it to public
String fileName = folderName + SUFFIX + "testvideo.mp4";
s3client.putObject(new PutObjectRequest(bucketName, fileName,
new File("C:\\Users\\user\\Desktop\\testvideo.mp4"))
.withCannedAcl(CannedAccessControlList.PublicRead));
deleteFolder(bucketName, folderName, s3client);
// deletes bucket
s3client.deleteBucket(bucketName);
}
public static void createFolder(String bucketName, String folderName, AmazonS3 client) {
// create meta-data for your folder and set content-length to 0
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(0);
// create empty content
InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
// create a PutObjectRequest passing the folder name suffixed by /
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName,
folderName + SUFFIX, emptyContent, metadata);
// send request to S3 to create folder
client.putObject(putObjectRequest);
}
/**
* This method first deletes all the files in given folder and than the
* folder itself
*/
public static void deleteFolder(String bucketName, String folderName, AmazonS3 client) {
List fileList =
client.listObjects(bucketName, folderName).getObjectSummaries();
for (S3ObjectSummary file : fileList) {
client.deleteObject(bucketName, file.getKey());
}
client.deleteObject(bucketName, folderName);
}
}
公用util：
package com.dianping.orderdish.dbh.hawkeye.util;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.*;
import com.dianping.lion.client.Lion;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.log4j.Logger;
import java.io.*;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
/**
* Created with Chen Zhe on 2017/8/31.
* Description:
*
* @author chenzhe
* @author chenzhe13@meituan.com
* @version 1.0.0
*/
public class AmazonS3Util {
private static final Logger LOGGER = Logger.getLogger(AmazonS3Util.class);
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");
private static final String DEFAULT_NAME = "dpManager.log";
private static final String BUCK_FROM = "dbh-log";
private static final String BUCK_TO = "dbh-plugins";
private static AmazonS3 s3Client;
public static class AmazonS3ClientProvider {
private static String accessKey = Lion.get("orderdish-dbh-netty-web.mss.private.accesskey");
private static String secretKey = Lion.get("orderdish-dbh-netty-web.mss.private.secretkey");
private static String url = Lion.get("orderdish-dbh-netty-web.mss.private.outurl", "http://mss.sankuai.com");
private AmazonS3ClientProvider() {
}
static AmazonS3 createAmazonS3Conn() throws IOException {
if (null != s3Client)
return s3Client;
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
ClientConfiguration clientConfig = new ClientConfiguration();
// clientConfig.setSignerOverride("S3SignerType");
URL ep = new URL(url);
if (ep.getProtocol().equalsIgnoreCase("http")) {
clientConfig.setProtocol(Protocol.HTTP);
} else if (ep.getProtocol().equalsIgnoreCase("https")) {
clientConfig.setProtocol(Protocol.HTTPS);
} else {
throw new IOException("Unsupported protocol");
}
String endpoint = ep.getHost();
if (ep.getPort() > 0) {
endpoint += ":" + ep.getPort();
}
S3ClientOptions s3FromOptions = new S3ClientOptions();
// mss only support path style access.
s3FromOptions.setPathStyleAccess(true);
s3Client = new AmazonS3Client(credentials, clientConfig);
s3Client.setS3ClientOptions(s3FromOptions);
s3Client.setEndpoint(endpoint);
// s3conn.setSignerRegionOverride("S3SignerType");
return s3Client;
}
}
public static String getDownloadURL(int shopId, String fileName) {
return getDownloadURL(shopId, fileName, BUCK_FROM);
}
public static String getDownloadURL(int shopId, String fileName, String buckName) {
try {
AmazonS3 s3From = AmazonS3ClientProvider.createAmazonS3Conn();
String key = getKey(shopId, fileName, buckName);
if (StringUtils.isBlank(key)) {
return null;
}
//生成一个可用于下载的URL
key = key.replace("/../", "%2F../");
GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(buckName, key);
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_MONTH, 1);
//设置过期时间
urlRequest.setExpiration(calendar.getTime());
//生成公用的url
URL url = s3From.generatePresignedUrl(urlRequest);
return url.toString();
} catch (IOException e) {
LOGGER.error("get download url error,shopId=" + shopId + ",fileName=" + fileName, e);
}
return null;
}
public static byte[] downloadFile(int shopId, String fileName) {
return downloadFile(shopId, fileName, BUCK_FROM);
}
public static byte[] downloadFile(int shopId, String fileName, String buckName) {
String tmpDir = System.getProperty("java.io.tmpdir");
Path zipFilePath = Paths.get(tmpDir, Thread.currentThread().toString() + "dpManager.zip");
ZipFile zipFile = null;
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
AmazonS3 s3From = AmazonS3ClientProvider.createAmazonS3Conn();
if (!new File(tmpDir).exists() && !new File(tmpDir).mkdir()) {
LOGGER.error("download file error because create file failed, " + tmpDir);
return null;
}
//Path unZipFilePath = Paths.get(tmpDir, "dpManager.log");
String key = getKey(shopId, fileName, buckName);
if (StringUtils.isBlank(key))
return null;
S3Object s3Object = s3From.getObject(buckName, key);
S3ObjectInputStream inputStream = s3Object.getObjectContent();
Files.copy(inputStream, zipFilePath, StandardCopyOption.REPLACE_EXISTING);
inputStream.close();
s3Object.close();
zipFile = new ZipFile(zipFilePath.toFile());
ZipEntry zipEntry = zipFile.entries().nextElement();
if (null == zipEntry || !zipEntry.getName().contains("dpManager")) {
LOGGER.error("download file error because can not find zipEntry");
return null;
}
InputStream zipFileStream = zipFile.getInputStream(zipEntry);
byte[] buf = new byte[1024];
while (zipFileStream.read(buf) != -1) {
outputStream.write(buf);
}
zipFileStream.close();
return outputStream.toByteArray();
} catch (IOException e) {
LOGGER.error("download file error,shopId=" + shopId + ",fileName=" + fileName, e);
} finally {
if (null != zipFile)
try {
zipFile.close();
new File(zipFilePath.toString()).delete();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
public static boolean upload(int shopId, String fileName, byte[] content) {
return upload(shopId, fileName, BUCK_FROM, BUCK_TO, content);
}
public static boolean upload(int shopId, String fileName,
String buckFrom, String buckTo,
byte[] content) {
try (InputStream inputStream = new ByteArrayInputStream(content)) {
AmazonS3 s3From = AmazonS3ClientProvider.createAmazonS3Conn();
String key = getKey(shopId, fileName, buckFrom);
if (StringUtils.isBlank(key))
return false;
ObjectMetadata metadataFrom = s3From.getObjectMetadata(buckFrom, key);
ObjectMetadata metadataTo = new ObjectMetadata();
metadataTo.setContentType(metadataFrom.getContentType());
metadataTo.setContentLength(metadataFrom.getContentLength());
s3From.putObject(buckTo, key.replace(buckFrom, buckTo), inputStream, metadataTo);
return true;
} catch (AmazonServiceException e) {
LOGGER.error("uploadFile AmazonServiceException", e);
} catch (AmazonClientException e) {
LOGGER.error("uploadFile AmazonClientException", e);
} catch (Exception e) {
LOGGER.error("uploadFile Exception", e);
}
return false;
}
private static String getKey(int shopId, String fileName) {
return getKey(shopId, fileName, BUCK_FROM);
}
private static String getKey(int shopId, String fileName, String buckName) {
try {
AmazonS3 s3From = AmazonS3ClientProvider.createAmazonS3Conn();
ObjectListing listing = s3From.listObjects(buckName, String.valueOf(shopId));
List<S3ObjectSummary> summaries = listing.getObjectSummaries();
Date logDate = getFileDate(fileName);
Date now = new Date();
for (S3ObjectSummary each : summaries) {
String key = each.getKey();
if (!key.split("/")[0].equals(String.valueOf(shopId))) {
continue;
}
if (!key.endsWith(fileName + ".zip") && !key.endsWith(fileName)) {
continue;
}
//当天如果拉了2次日志,避免第二次日志上传还没结束,就去下载上一次上传的日志
//eg: 10:00上传一次日志 14:00在下载日志时,10:00的日志已经存在,即便日志还没上传完成,也能生成一个URL(老的日志)
//对于非当天日志，如果最后修改时间大于文件日志，认为文件是完整的，不需要重新等待上传
Date lastModified = each.getLastModified();
if (DateUtils.isSameDay(logDate, now)) {
if (DateUtils.addMinutes(lastModified, 10).compareTo(now) < 0) {
continue;
}
} else if (lastModified.before(DateUtils.addDays(logDate, 1))) {
continue;
}
//String keyTo = key.replace(BUCK_FROM, BUCK_TO);
//AmazonS3ClientProvider.createAmazonS3Conn().copyObject(BUCK_FROM, key, BUCK_TO, keyTo);
//LOGGER.info(String.format("copy file from %s to %s",key, keyTo));
return key;
}
} catch (IOException e) {
LOGGER.error("getKey failed, shopId=" + shopId + ",fileName=" + fileName, e);
}
return null;
}
private static Date getFileDate(String logFileName) {
if (logFileName.equalsIgnoreCase(DEFAULT_NAME))
return new Date();
try {
return DATE_FORMAT.parse(logFileName.substring(14, 24));
} catch (ParseException e) {
LOGGER.error("getFileDate failed, fileName is " + logFileName, e);
}
return new Date();
}
}参考：
https://javatutorial.net/java-s3-example https://aws.amazon.com/cn/s3/getting-started/
python版的s3 api：
http://boto3.readthedocs.io/en/latest/guide/quickstart.html
内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 
相关文章推荐
- 《连载 | 物联网框架ServerSuperIO教程》- 14．配制工具介绍，以及设备驱动、视图驱动、服务实例的挂载
- python数字类型介绍以及创建数字值对象引用、删除数字值引用操作实例
- Oracle 阻塞(blocking blocked)介绍和实例演示
- EBS VPD介绍和使用实例
- Protocol Buffer介绍（Java实例)
- java.util.Collections.synchronizedMap()方法实例
- 实例介绍C# GUI开发
- sql事务（Transaction）用法介绍及回滚实例
- redis学习及实践3---Jedis、JedisPool、Jedis分布式实例介绍
- [Python]网络爬虫（四）：Opener与Handler的介绍和实例应用
- 实例详细介绍各种字符集编码转换问题 [转]
- 实例介绍Cocos2d-x物理引擎：HelloPhysicsWorld
- java.util.UUID通用唯一识别码的介绍
- [转] 引用 Java自带的线程池ThreadPoolExecutor详细介绍说明和实例应用
- linux awk 内置函数详细介绍（实例）
- SSL介绍与Java实例
- 对javascript基本对象的属性以及方法的实例介绍
- 全面介绍Windows内存管理机制及C++内存分配实例（二）：内存状态查询
- java.util.Arrays介绍
- 实例介绍 Java(android) 回调函数使用方法