作为AWS众多云服务的核心成员之一，S3得到了非常广泛的应用。下面来介绍一下如何通过AWS S3。本次主要介绍如何在Node.js中访问S3中的资源，比如创建Bucket，上传文件到S3，列举S3中的文件。
AWS S3
创建S3 Bucket
Javascript
const AWS = require("aws-sdk");
AWS.config.update({ region: "eu-west-1" });
s3 = new AWS.S3();
let bucketParams = {
Bucket: 'lcoding-s3-demo',
};
s3.createBucket(bucketParams, function(err, data) {
if(err) {
console.log("Error", err);
} else {
console.log("Created", data.Location);
}
});上传文件到S3 Bucket
Javascript
const AWS = require("aws-sdk");
const fs = require("fs");
const path = require("path");
AWS.config.update({ region: "eu-west-1" });
s3 = new AWS.S3();
let params = {
Bucket: 'lcoding-s3-demo',
Key: "",
Body: ""
};
let fileName = "test.txt"
let file = fs.createReadStream(fileName);
file.on("error", function(err) {
console.log("File Error", err);
});
params.Body = file;
params.Key = path.basename(fileName);
s3.upload(params, function(err, data) {
if(err) {
console.log("Error", err);
} else {
console.log("Uploaded", data.Location);
}
});获取S3 Bucket信息
Javascript
const AWS = require("aws-sdk");
AWS.config.update({ region: "eu-west-1" });
s3 = new AWS.S3();
let params = {
Bucket: 'lcoding-s3-demo'
};
s3.listObjects(params, function(err, data) {
if(err) {
console.log("Error", err);
} else {
console.log("Created", data);
}
});获取一个S3文件对象及其内容
Javascript
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
async function getObject (bucket, objectKey) {
try {
const params = {
Bucket: bucket,
Key: objectKey
}
const data = await s3.getObject(params).promise();
console.log(data);
return data.Body.toString('utf-8');
} catch (e) {
throw new Error(`Could not retrieve file from S3: ${e.message}`)
}
}
// To retrieve you need to use `await getObject()` or `getObject().then()`
getObject('lcoding-s3-demo', 'test.txt').then((err, data) => {
if(err) {
console.log(err);
}else if(data) {
console.log(data);
}
});其中这行：
Javascript
console.log(data);的输出为文件对象的元数据：
json
{
AcceptRanges: 'bytes',
LastModified: 2021-11-27T21:26:03.000Z,
ContentLength: 10,
ETag: '"4606722a4810cf747aa8adead38295fa"',
ContentType: 'application/octet-stream',
Metadata: {},
Body: <Buffer 31 32 33 0d 0a 34 35 36 0d 0a>
} 检查一个文件是否存在
TypeScript
async exist(bucketName: string, fileName: string) {
const params = {
Bucket: bucketName,
Key: fileName,
};
try {
const data = await s3Client.headObject(params).promise();
if (data.ContentLength && data.ContentLength > 0) {
return true;
}
return false;
} catch (error) {
return false;
}
}复制文件
TypeScript
async copy(source: string, bucketName: string, fileName: string) {
const params = {
Bucket: bucketName,
CopySource: source,
Key: fileName,
};
const response = await s3Client.copyObject(params).promise();
if (!response) {
throw Error('there was an error copying the file');
}
return response;
},获取presignedUrl
TypeScript
async getPreSignedUrl(bucketName: string, fileName: string, mime: string) {
const s3Params = {
Bucket: bucketName,
Key: fileName,
ContentType: mime,
};
const result = await s3Client.getSignedUrlPromise('putObject', s3Params);
return result;
},通过流方式创建zip文件
- 目录结构：YOUR_BUCKET/current/STUDENT_ID/
- 源文件：YOUR_BUCKET/current/STUDENT_ID/*
- 目标文件：YOUR_BUCKET/current/STUDENT_ID/STUDENT_ID.zip
JavaScript
import AWS from 'aws-sdk';
import { Stream } from 'stream';
import archiver from 'archiver';
import S3 from 'common/S3';
const s3Client = new AWS.S3();
const createZipFile = async (studentId: string, overwrite = true) => {
try {
const fileBucketName = process.env.FILE_BUCKET as string;
const foundZipFile = await S3.exist(fileBucketName, `current/${studentId}/${studentId}.zip`);
if (foundZipFile && !overwrite) {
console.log(`Found zip file, bypass: current/${studentId}/${studentId}.zip`);
return;
}
console.log(`Generating zip file: current/${studentId}/${studentId}.zip`);
const params = {
Bucket: fileBucketName,
Delimiter: '/',
Prefix: `current/${studentId}/`,
};
s3Client.listObjects(params, async (err, data) => {
if (err) throw err;
const files: { fileName: string; key: string }[] = [];
data.Contents?.forEach(fileInfo => {
const currentFileName = fileInfo.Key?.split('/').pop() as string;
files.push({
fileName: currentFileName,
key: fileInfo.Key as string,
});
});
const zipParams = {
files,
zippedFileKey: `current/${studentId}/${studentId}.zip`,
};
const s3FileDownloadStreams = zipParams.files.map(item => {
const stream = s3Client
.getObject({ Bucket: fileBucketName, Key: item.key })
.createReadStream();
return {
stream,
fileName: item.fileName,
};
});
const streamPassThrough = new Stream.PassThrough();
const uploadParams = {
Bucket: fileBucketName,
ACL: 'public-read',
Body: streamPassThrough,
ContentType: 'application/zip',
Key: zipParams.zippedFileKey,
};
const s3Upload = s3Client.upload(uploadParams, (uploadErr: any, uploadData: any) => {
if (uploadErr) console.error('upload error', uploadErr);
else console.log('upload done', uploadData);
});
const archive = archiver('zip', {
zlib: { level: 5 },
});
archive.on('error', error => {
throw new Error(`${error.name} ${error.code} ${error.message} ${error.path} ${error.stack}`);
});
s3Upload.on('httpUploadProgress', progress => {
console.log(progress);
});
await new Promise((resolve, reject) => {
archive.pipe(streamPassThrough);
s3FileDownloadStreams.forEach(s3FileDownloadStream => {
archive.append(s3FileDownloadStream.stream, {
name: s3FileDownloadStream.fileName,
});
});
archive.finalize();
}).catch(error => {
throw new Error(`${error.code} ${error.message} ${error.data}`);
});
});
} catch (error) {
console.log(studentId, 'error', error);
}
};
export default createZipFile;
