我的项目背景
前端开发过程中经常须要用到的图片等资源，除了应用常见的第三方图床外，咱们也能够本人搭建一个公有图床，为团队提供前端根底服务。本文旨在回顾总结下自建图床的后端局部实现计划，心愿可能给有相似需要的同学一些借鉴和计划。另外说一下，因为是前端根底建设，这里咱们齐全由前端同学所相熟的node.js来实现所须要的后端服务需要。
计划
后端局部架构选型，因为这里次要是为前端业务开发人员提供基建服务，而团体平台也提供了各种云服务，并且并不会呈现过多的高并发等场景，因此在语言选择上还是以前端同学所相熟的node.js为主，这里咱们团队次要以express框架为主，在整个大的专网技术团队中，后端依然以java为主，node次要作为中间层BFF来对局部接口进行开发聚合等，因此主体依然以单体架构为主，微服务模式则采纳service mesh的云服务产品（如：istio）来和java同学进行配合，而没有采纳一些node.js的微服务框架（比方：nest.js中有微服务相干的设置，以及seneca等）。因为是单体利用，鉴于express的中间件机制，通过路由对不同模块进行了拆散，本图床服务中提供的服务都隔离在imagepic的模块下；在数据库抉择方面，图床这里仅仅须要一个鉴权机制，其余并没有特地额定的长久化需要，这里我抉择了mongodb作为数据库长久化数据（ps：云中间件提供的mongodb呈现了接入问题，后续通过CFS（文件存储系统）+FaaS来实现了代替计划）；因为图床性能的特殊性，对于上传图片进行了流的转换，这里会用到一个长期图片存储的过程，通过云产品的CFS（文件存储系统）来进行长久化存储，定期进行数据的删除；而真正的图片存储则是放在了COS（对象存储）中，相较于CFS的文件接口标准，COS则是基于亚马逊的S3标准的，因此这里更适宜于作为图片的存储载体
目录
db
- \_\_temp\_\_
- imagepic
deploy
dev
- Dockerfile
- pv.yaml
- pvc.yaml
- server.yaml
production
- Dockerfile
- pv.yaml
- pvc.yaml
- server.yaml
- build.sh
faas
- index.js
- model.js
- operator.js
- read.js
- utils.js
- write.js
server
api
- openapi.yaml
lib
- index.js
- cloud.js
- jwt.js
- mongodb.js
routes
imagepic
- auth
- bucket
- notification
- object
- policy
- index.js
- minio.js
- router.js
utils
- index.js
- is.js
- pagination.js
- reg.js
- uuid.js
- app.js
- config.js
- index.js
- main.js
实际
对波及到局部接口须要进行鉴权判断，这里应用的是jwt进行相干的权限校验
源码
faas
这里形象进去了云函数来为后端服务提供能力，模仿实现相似mongodb相干的一些数据库操作
model.js
定义的model相干的数据格式
/** * documents 数据结构 * @params * _name String 文件的名称 * _collections Array 文件的汇合 * @examples * const documents = { * "_name": String, * "_collections": Array * } */exports.DOCUMENTS_SCHEMA = { "_name": String, "_collections": Array}/** * collections 数据结构 * @params * _id String 汇合的默认id * _v Number 汇合的自增数列 * @examples * const collections = { * "_id": String, * "_v": Number, * } */ exports.COLLECTIONS_SCHEMA = { "_id": String}read.js
node的fs模块读文件操作
const { isExit, genCollection, genDocument, findCollection, findLog, stringify, fs, compose, path} = require('./utils');exports.read = async (method, ...args) => { let col = '', log = ''; const isFileExit = isExit(args[0], `${args[1]}_${args[2]['phone']}.json`); console.log('isFileExit', isFileExit) const doc = genDocument(...args); switch (method) { case 'FIND': col = compose( stringify, findCollection )(doc, genCollection(...args)); log = compose( stringify, findLog, genCollection )(...args); break; }; if(isFileExit) { return fs.promises.readFile(path.resolve(__dirname, `../db/${args.slice(0,2).join('/')}_${args[2][`phone`]}.json`), {encoding: 'utf-8'}).then(res => { console.log('res', res); console.log(log) return { flag: true, data: res, }; }) } else { return { flag: false, data: {} }; }};write.js
node的fs模块的写文件操作
const { isExit, fs, path, stringify, compose, genCollection, addCollection, addLog, updateCollection, updateLog, removeCollection, removeLog, genDocument} = require('./utils');exports.write = async (method, ...args) => { console.log('write args', args, typeof args[2]); const isDirExit = isExit(args.slice(0, 1)); const doc = genDocument(...args); let col = '', log = ''; switch (method) { case 'ADD': col = compose( stringify, addCollection )(doc, genCollection(...args)); log = compose( stringify, addLog, genCollection )(...args); break; case 'REMOVE': col = compose( stringify, removeCollection )(doc, genCollection(...args)); log = compose( stringify ,removeLog, genCollection )(...args); break; case 'UPDATE': col = compose( stringify, updateCollection )(doc, genCollection(...args)); log = compose( stringify, updateLog, genCollection )(...args); break; } if (!isDirExit) { return fs.promises.mkdir(path.resolve(__dirname, `../db/${args[0]}`)) .then(() => { console.log(`创立数据库${args[0]}胜利`); return true; }) .then(flag => { if (flag) { return fs.promises.writeFile(path.resolve(__dirname, `../db/${args.slice(0,2).join('/')}_${args[2][`phone`]}.json`), col) .then(() => { console.log(log); return true; }) .catch(err => console.error(err)) } }) .catch(err => console.error(err)) } else { return fs.promises.writeFile(path.resolve(__dirname, `../db/${args.slice(0,2).join('/')}_${args[2][`phone`]}.json`), col) .then(() => { console.log(log) return true; }) .catch(err => console.error(err)) }};operator.js
const { read } = require('./read');const { write } = require('./write');exports.find = async (...args) => await read('FIND', ...args);exports.remove = async (...args) => await write('REMOVE', ...args);exports.add = async (...args) => await write('ADD', ...args);exports.update = async (...args) => await write('UPDATE', ...args);utils.js
共用工具包
const { DOCUMENTS_SCHEMA, COLLECTIONS_SCHEMA } = require('./model');const { v4: uuidv4 } = require('uuid');const path = require('path');const fs = require('fs');exports.path = path;exports.uuid = uuidv4;exports.fs = fs;exports.compose = (...funcs) => { if(funcs.length===0){ return arg=>arg; } if(funcs.length===1){ return funcs[0]; } return funcs.reduce((a,b)=>(...args)=>a(b(...args)));};exports.stringify = arg => JSON.stringify(arg);exports.isExit = (...args) => fs.existsSync(path.resolve(__dirname, `../db/${args.join('/')}`));console.log('DOCUMENTS_SCHEMA', DOCUMENTS_SCHEMA);exports.genDocument = (...args) => { return { _name: args[1], _collections: [] }};console.log('COLLECTIONS_SCHEMA', COLLECTIONS_SCHEMA);exports.genCollection = (...args) => { return { _id: uuidv4(), ...args[2] }};exports.addCollection = ( doc, col ) => { doc._collections.push(col); return doc;};exports.removeCollection = ( doc, col ) => { for(let i = 0; i < doc._collections.length; i++) { if(doc._collections[i][`_id`] == col._id) { doc._collections.splice(i,1) } } return doc;};exports.findCollection = ( doc, col ) => { return doc._collections.filter(f => f._id == col._id)[0];};exports.updateCollection = ( doc, col ) => { doc._collections = [col]; return doc;};exports.addLog = (arg) => { return `减少了汇合 ${JSON.stringify(arg)}`};exports.removeLog = () => { return `移除汇合胜利`};exports.findLog = () => { return `查问汇合胜利`};exports.updateLog = (arg) => { return `更新了汇合 ${JSON.stringify(arg)}`};lib
cloud.js
业务操作应用云函数
const { find, update, remove, add} = require('../../faas');exports.cloud_register = async (dir, file, params) => { const findResponse = await find(dir, file, params); if (findResponse.flag) { return { flag: false, msg: '已注册' } } else { const r = await add(dir, file, params); console.log('cloud_register', r) if (r) { return { flag: true, msg: '胜利' } } else { return { flag: false, msg: '失败' } } }}exports.cloud_login = async (dir, file, params) => { const r = await find(dir, file, params); console.log('cloud_read', r) if (r.flag == true) { if (JSON.parse(r.data)._collections[0].upwd === params.upwd) { return { flag: true, msg: '登录胜利' } } else { return { flag: false, msg: '明码不正确' } } } else { return { flag: false, msg: '失败' } }}exports.cloud_change = async (dir, file, params) => { const r = await update(dir, file, params); console.log('cloud_change', r) if (r) { return { flag: true, msg: '批改明码胜利' } } else { return { flag: false, msg: '失败' } }}jwt.js
jwt验证相干配置
const jwt = require('jsonwebtoken');const { find} = require('../../faas');exports.jwt = jwt;const expireTime = 60 * 60;exports.signToken = (rawData, secret) => { return jwt.sign(rawData, secret, { expiresIn: expireTime });};exports.verifyToken = (token, secret) => { return jwt.verify(token, secret, async function (err, decoded) { if (err) { console.error(err); return { flag: false, msg: err } } console.log('decoded', decoded, typeof decoded); const { phone, upwd } = decoded; let r = await find('imagepic', 'auth', { phone, upwd }); console.log('r', r) if (r.flag == true) { if (JSON.parse(r.data)._collections[0].upwd === decoded.upwd) { return { flag: true, msg: '验证胜利' } } else { return { flag: false, msg: '登录明码不正确' } } } else { return { flag: false, msg: '登录用户未找到' } } });}auth
用于登录注册验证
const router = require('../../router');const url = require('url');const { pagination, isEmpty, isArray, PWD_REG, NAME_REG, EMAIL_REG, PHONE_REG} = require('../../../utils');const { // mongoose, cloud_register, cloud_login, cloud_change, signToken} = require('../../../lib');// const Schema = mongoose.Schema;/** * @openapi * /imagepic/auth/register: post: summary: 注册 tags: - listObjects requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/register' responses: '200': content: application/json: example: code: "0" data: {} msg: "胜利" success: true */router.post('/register', async function (req, res) { const params = req.body; console.log('params', params); let flag = true, err = []; const { name, tfs, email, phone, upwd } = params; flag = flag && PWD_REG.test(upwd) && EMAIL_REG.test(email) && PHONE_REG.test(phone); if (!PWD_REG.test(upwd)) err.push('明码不符合规范'); if (!EMAIL_REG.test(email)) err.push('邮箱填写不符合规范'); if (!PHONE_REG.test(phone)) err.push('手机号码填写不符合规范'); // const registerSchema = new Schema({ // name: String, // tfs: String, // email: String, // phone: String, // upwd: String // }); // const Register = mongoose.model('Register', registerSchema); if (flag) { // const register = new Register({ // name, // tfs, // email, // phone, // upwd // }); // register.save().then((result)=>{ // console.log("胜利的回调", result); // res.json({ // code: "0", // data: {}, // msg: '胜利', // success: true // }); // },(err)=>{ // console.log("失败的回调", err); // res.json({ // code: "-1", // data: { // err: err // }, // msg: '失败', // success: false // }); // }); let r = await cloud_register('imagepic', 'auth', { name, tfs, email, phone, upwd }); if (r.flag) { res.json({ code: "0", data: {}, msg: '胜利', success: true }); } else { res.json({ code: "-1", data: { err: r.msg }, msg: '失败', success: false }); } } else { res.json({ code: "-1", data: { err: err.join(',') }, msg: '失败', success: false }) }});/** * @openapi * /imagepic/auth/login: post: summary: 登录 tags: - listObjects requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/login' responses: '200': content: application/json: example: code: "0" data: {token:'xxx'} msg: "胜利" success: true */router.post('/login', async function (req, res) { const params = req.body; console.log('params', params); let flag = true, err = []; const { phone, upwd } = params; flag = flag && PWD_REG.test(upwd) && PHONE_REG.test(phone); if (!PWD_REG.test(upwd)) err.push('明码不符合规范'); if (!PHONE_REG.test(phone)) err.push('手机号码填写不符合规范'); // const registerSchema = new Schema({ // name: String, // tfs: String, // email: String, // phone: String, // upwd: String // }); // const Register = mongoose.model('Register', registerSchema); if (flag) { // const register = new Register({ // name, // tfs, // email, // phone, // upwd // }); // register.save().then((result)=>{ // console.log("胜利的回调", result); // res.json({ // code: "0", // data: {}, // msg: '胜利', // success: true // }); // },(err)=>{ // console.log("失败的回调", err); // res.json({ // code: "-1", // data: { // err: err // }, // msg: '失败', // success: false // }); // }); let r = await cloud_login('imagepic', 'auth', { phone, upwd }); if (r.flag) { const token = signToken({ phone, upwd }, 'imagepic'); // console.log('token', token) res.json({ code: "0", data: { token: token }, msg: '胜利', success: true }); } else { res.json({ code: "-1", data: { err: r.msg }, msg: '失败', success: false }); } } else { res.json({ code: "-1", data: { err: err.join(',') }, msg: '失败', success: false }) }});/** * @openapi * /imagepic/auth/change: post: summary: 批改明码 tags: - listObjects requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/change' responses: '200': content: application/json: example: code: "0" data: {token:'xxx'} msg: "胜利" success: true */router.post('/change', async function (req, res) { const params = req.body; console.log('params', params); let flag = true, err = []; const { phone, opwd, npwd } = params; flag = flag && PWD_REG.test(opwd) && PWD_REG.test(npwd) && PHONE_REG.test(phone); if (!PWD_REG.test(opwd)) err.push('旧明码不符合规范'); if (!PWD_REG.test(npwd)) err.push('新密码不符合规范'); if (!PHONE_REG.test(phone)) err.push('手机号码填写不符合规范'); if (flag) { let r = await cloud_login('imagepic', 'auth', { phone: phone, upwd: opwd }); if (r.flag) { const changeResponse = await cloud_change('imagepic', 'auth', { phone: phone, upwd: npwd }); if(changeResponse.flag) { res.json({ code: "0", data: {}, msg: '胜利', success: true }); } else { res.json({ code: "-1", data: { err: changeResponse.msg }, msg: '失败', success: false }); } } else { res.json({ code: "-1", data: { err: r.msg }, msg: '失败', success: false }); } } else { res.json({ code: "-1", data: { err: err.join(',') }, msg: '失败', success: false }) }})module.exports = router;bucket
桶操作相干的接口
const minio = require('../minio');const router = require('../../router');const url = require('url');const { pagination, isEmpty, isArray} = require('../../../utils');/** * @openapi * /imagepic/bucket/listBuckets: summary: 查问所有存储桶 get: parameters: - name: pageSize name: pageNum in: query description: user id. required: false tags: - List responses: '200': content: application/json: example: code: "0" data: [ { "name": "5g-fe-file", "creationDate": "2021-06-04T10:01:42.664Z" }, { "name": "5g-fe-image", "creationDate": "2021-05-28T01:34:50.375Z" } ] message: "胜利" success: true */router.get('/listBuckets', function (req, res) { const params = url.parse(req.url, true).query; console.log('params', params); minio.listBuckets(function (err, buckets) { if (err) return console.log(err) // console.log('buckets :', buckets); res.json({ code: "0", // 分页解决 data: isEmpty(params) ? buckets : isArray(buckets) ? ( params.pageSize && params.pageNum ) ? pagination(buckets, params.pageSize, params.pageNum) : [] : [], msg: '胜利', success: true }) })})module.exports = router;object
用于图片对象相干的接口
const minio = require('../minio');const router = require('../../router');const multer = require('multer');const path = require('path');const fs = require('fs');const { pagination} = require('../../../utils');const { verifyToken} = require('../../../lib');/** * @openapi * /imagepic/object/listObjects: get: summary: 获取存储桶中的所有对象 tags: - listObjects requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/listObjects' responses: '200': content: application/json: example: code: "0" data: 49000 msg: "胜利" success: true */router.post('/listObjects', function (req, res) { const params = req.body; // console.log('listObjects params', params) const { bucketName, prefix, pageSize, pageNum } = params; const stream = minio.listObjects(bucketName, prefix || '', false) let flag = false, data = []; stream.on('data', function (obj) { data.push(obj); flag = true; }) stream.on('error', function (err) { console.log(err) data = err; flag = false; }) stream.on('end', function (err) { if (flag) { // 分页解决 res.json({ code: "0", data: pageNum == -1 ? { total: data.length, lists: data } : { total: data.length, lists: pagination(data, pageSize || 10, pageNum || 1) }, msg: '胜利', success: true }) } else { res.json({ code: "-1", data: err, msg: '失败', success: false }) } })})/** * @openapi * /imagepic/object/getObject: post: summary: 下载对象 tags: - getObject requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/getObject' responses: '200': content: application/json: example: code: "0" data: 49000 msg: "胜利" success: true */router.post('/getObject', function (req, res) { const params = req.body; // console.log('statObject params', params) const { bucketName, objectName } = params; minio.getObject(bucketName, objectName, function (err, dataStream) { if (err) { return console.log(err) } let size = 0; dataStream.on('data', function (chunk) { size += chunk.length }) dataStream.on('end', function () { res.json({ code: "0", data: size, msg: '胜利', success: true }) }) dataStream.on('error', function (err) { res.json({ code: "-1", data: err, msg: '失败', success: false }) }) })})/** * @openapi * /imagepic/object/statObject: post: summary: 获取对象元数据 tags: - statObject requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/statObject' responses: '200': content: application/json: example: code: "0" data: { "size": 47900, "metaData": { "content-type": "image/png" }, "lastModified": "2021-10-14T07:24:59.000Z", "versionId": null, "etag": "c8a447108f1a3cebe649165b86b7c997" } msg: "胜利" success: true */router.post('/statObject', function (req, res) { const params = req.body; // console.log('statObject params', params) const { bucketName, objectName } = params; minio.statObject(bucketName, objectName, function (err, stat) { if (err) { return console.log(err) } // console.log(stat) res.json({ code: "0", data: stat, msg: '胜利', success: true }) })})/** * @openapi * /imagepic/object/presignedGetObject: post: summary: 获取对象长期连贯 tags: - presignedGetObject requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/presignedGetObject' responses: '200': content: application/json: example: code: "0" data: "http://172.24.128.7/epnoss-antd-fe/b-ability-close.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=7RGX0TJQE5OX9BS030X6%2F20211126%2Fdefault%2Fs3%2Faws4_request&X-Amz-Date=20211126T031946Z&X-Amz-Expires=604800&X-Amz-SignedHeaders=host&X-Amz-Signature=27644907283beee2b5d6f468ba793db06cd704e7b3fb1c334f14665e0a8b6ae4" msg: "胜利" success: true */router.post('/presignedGetObject', function (req, res) { const params = req.body; // console.log('statObject params', params) const { bucketName, objectName, expiry } = params; minio.presignedGetObject(bucketName, objectName, expiry || 7 * 24 * 60 * 60, function (err, presignedUrl) { if (err) { return console.log(err) } // console.log(presignedUrl) res.json({ code: "0", data: presignedUrl, msg: '胜利', success: true }) })})/** * @openapi * /imagepic/object/putObject: post: summary: 上传图片 tags: - putObject requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/putObject' responses: '200': content: application/json: example: code: "0" data: "" msg: "胜利" success: true */router.post('/putObject', multer({ dest: path.resolve(__dirname, '../../../../db/__temp__')}).single('file'), async function (req, res) { console.log('/putObject', req.file, req.headers); const verifyResponse = await verifyToken(req.headers.authorization, 'imagepic'); console.log('verifyResponse', verifyResponse) const bucketName = req.headers.bucket, folder = req.headers.folder, originName = req.file['originalname'], file = req.file['path'], ext = path.extname(req.file['originalname']), fileName = req.file['filename']; console.log('folder', folder); if (!verifyResponse.flag) { fs.unlink(path.resolve(__dirname, `../../../../db/__temp__/${fileName}`), function (err) { if (err) { console.error(`删除文件 ${fileName} 失败，失败起因：${err}`) } console.log(`删除文件 ${fileName} 胜利`) }); return res.json({ code: "-1", data: verifyResponse.msg, msg: '未满足权限', success: false }) } else { const fullName = folder ? `${folder}/${originName}` : `${originName}`; fs.stat(file, function (err, stats) { if (err) { return console.log(err) } minio.putObject(bucketName, fullName, fs.createReadStream(file), stats.size, { 'Content-Type': `image/${ext}` }, function (err, etag) { fs.unlink(path.resolve(__dirname, `../../../../db/__temp__/${fileName}`), function (err) { if (err) { console.error(`删除文件 ${fileName} 失败，失败起因：${err}`) } console.log(`删除文件 ${fileName} 胜利`) }); if (err) { return res.json({ code: "-1", data: err, msg: '失败', success: false }) } else { return res.json({ code: "0", data: etag, msg: '胜利', success: true }) } }) }) }});/** * @openapi * /imagepic/object/removeObject: post: summary: 删除图片 tags: - removeObject requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/removeObject' responses: '200': content: application/json: example: code: "0" data: "" msg: "胜利" success: true */router.post('/removeObject', async function (req, res) { console.log('/removeObject', req.body, req.headers); const verifyResponse = await verifyToken(req.headers.authorization, 'imagepic'); if (!verifyResponse.flag) { return res.json({ code: "-1", data: verifyResponse.msg, msg: '未满足权限', success: false }) } else { const { bucketName, objectName } = req.body; minio.removeObject(bucketName, objectName, function (err) { if (err) { return res.json({ code: "-1", data: err, msg: '失败', success: false }) } return res.json({ code: "0", data: {}, msg: '胜利', success: true }) }) }});module.exports = router;总结
在针对前端图床的后端接口开发过程中，切实感触到应用Serverless形式进行数据侧开发的简略，对于node.js来说更好的应用faas模式进行相干的函数粒度的业务开发可能更加有实用场景，而对于其余目前已有的一些其余场景，node.js在后端市场中其实很难撼动java、go、c++等传统后端语言的位置的，因此集体认为在某些场景，比方重IO以及事件模型为主的业务中，node.js的Serverless化可能会成为后续发展势头，配合其余重计算场景的多语言后端服务模式或者才是将来的一种状态。（ps：这里只是用到了faas这么一个概念，真正的Serverless不应该仅仅是用到了这么一个函数的业态，更重要的对于baas层的调度才是服务端更应该重视的，是不是Serverless无所谓，咱们次要关注的应该是服务而不是资源）