worldcone/cherrypick
2
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

feat: Enable storage of remote file cache in a separate object storage

Browse source
This commit is contained in:
caipira113 2023-07-17 21:32:22 +09:00 committed by NoriDev
parent 9c6d6d1d4c
commit 272a124e45
11 changed files with 483 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
packages/backend/migration/1689580926821-ObjectStorageRemoteSetting.js Normal file
View file

@ -0,0 +1,36 @@
export class ObjectStorageRemoteSetting1689580926821 {
name = 'ObjectStorageRemoteSetting1689580926821'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "useObjectStorageRemote" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteBucket" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemotePrefix" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteBaseUrl" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteEndpoint" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteRegion" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteAccessKey" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteSecretKey" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemotePort" integer`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteUseSSL" boolean NOT NULL DEFAULT true`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteUseProxy" boolean NOT NULL DEFAULT true`, undefined);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteSetPublicRead" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteS3ForcePathStyle" boolean NOT NULL DEFAULT true`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteS3ForcePathStyle"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteSetPublicRead"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteUseProxy"`, undefined);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteUseSSL"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemotePort"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteSecretKey"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteAccessKey"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteRegion"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteEndpoint"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteBaseUrl"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemotePrefix"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteBucket"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "useObjectStorageRemote"`);
}
}

58
packages/backend/src/core/DriveService.ts
View file

@ -131,9 +131,10 @@ export class DriveService {
* @param type Content-Type for original
* @param hash Hash for original
* @param size Size for original
* @param isRemote If true, file is remote file
*/
@bindThis
private async save(file: DriveFile, path: string, name: string, type: string, hash: string, size: number): Promise<DriveFile> {
private async save(file: DriveFile, path: string, name: string, type: string, hash: string, size: number, isRemote: boolean): Promise<DriveFile> {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);
@ -158,11 +159,19 @@ export class DriveService {
ext = '';
}
const baseUrl = meta.objectStorageBaseUrl
?? `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`;
const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;
const objectStorageBaseUrl = useObjectStorageRemote ? meta.objectStorageRemoteBaseUrl : meta.objectStorageBaseUrl;
const objectStorageUseSSL = useObjectStorageRemote ? meta.objectStorageRemoteUseSSL : meta.objectStorageUseSSL;
const objectStorageEndpoint = useObjectStorageRemote ? meta.objectStorageRemoteEndpoint : meta.objectStorageEndpoint;
const objectStoragePort = useObjectStorageRemote ? meta.objectStorageRemotePort : meta.objectStoragePort;
const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket;
const objectStoragePrefix = useObjectStorageRemote ? meta.objectStorageRemotePrefix : meta.objectStoragePrefix;
const baseUrl = objectStorageBaseUrl
?? `${ objectStorageUseSSL ? 'https' : 'http' }://${ objectStorageEndpoint }${ objectStoragePort ? `:${objectStoragePort}` : '' }/${ objectStorageBucket }`;
// for original
const key = `${meta.objectStoragePrefix}/${randomUUID()}${ext}`;
const key = `${objectStoragePrefix}/${randomUUID()}${ext}`;
const url = `${ baseUrl }/${ key }`;
// for alts
@ -175,23 +184,23 @@ export class DriveService {
//#region Uploads
this.registerLogger.info(`uploading original: ${key}`);
const uploads = [
this.upload(key, fs.createReadStream(path), type, null, name),
this.upload(key, fs.createReadStream(path), type, isRemote, null, name),
];
if (alts.webpublic) {
webpublicKey = `${meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicKey = `${objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;
this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, isRemote, alts.webpublic.ext, name));
}
if (alts.thumbnail) {
thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailKey = `${objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;
this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`));
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, isRemote, alts.thumbnail.ext, `${name}.thumbnail`));
}
await Promise.all(uploads);
@ -360,14 +369,18 @@ export class DriveService {
* Upload to ObjectStorage
*/
@bindThis
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, ext?: string | null, filename?: string) {
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, isRemote: boolean, ext?: string | null, filename?: string) {
if (type === 'image/apng') type = 'image/png';
if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream';
const meta = await this.metaService.fetch();
const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;
const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket;
const objectStorageSetPublicRead = useObjectStorageRemote ? meta.objectStorageRemoteSetPublicRead : meta.objectStorageSetPublicRead;
const params = {
Bucket: meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
Body: stream,
ContentType: type,
@ -380,9 +393,9 @@ export class DriveService {
// 許可されているファイル形式でしか拡張子をつけない
ext ? correctFilename(filename, ext) : filename,
);
if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
if (objectStorageSetPublicRead) params.ACL = 'public-read';
await this.s3Service.upload(meta, params)
await this.s3Service.upload(meta, params, isRemote)
.then(
result => {
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
@ -618,7 +631,8 @@ export class DriveService {
}
}
} else {
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size));
const isRemote = user ? this.userEntityService.isRemoteUser(user) : false;
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size, isRemote));
}
this.registerLogger.succ(`drive file has been created ${file.id}`);
@ -672,7 +686,7 @@ export class DriveService {
}
@bindThis
public async deleteFileSync(file: DriveFile, isExpired = false) {
public async deleteFileSync(file: DriveFile, isExpired = false, isRemote: boolean) {
if (file.storedInternal) {
this.internalStorageService.del(file.accessKey!);
@ -686,14 +700,14 @@ export class DriveService {
} else if (!file.isLink) {
const promises = [];
promises.push(this.deleteObjectStorageFile(file.accessKey!));
promises.push(this.deleteObjectStorageFile(file.accessKey!, isRemote));
if (file.thumbnailUrl) {
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!));
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!, isRemote));
}
if (file.webpublicUrl) {
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!));
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!, isRemote));
}
await Promise.all(promises);
@ -733,15 +747,17 @@ export class DriveService {
}
@bindThis
public async deleteObjectStorageFile(key: string) {
public async deleteObjectStorageFile(key: string, isRemote: boolean) {
const meta = await this.metaService.fetch();
const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;
const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket;
try {
const param = {
Bucket: meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
} as DeleteObjectCommandInput;
await this.s3Service.delete(meta, param);
await this.s3Service.delete(meta, param, isRemote);
} catch (err: any) {
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);

44
packages/backend/src/core/S3Service.ts
View file

@ -23,35 +23,45 @@ export class S3Service {
}
@bindThis
public getS3Client(meta: Meta): S3Client {
const u = meta.objectStorageEndpoint
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
public getS3Client(meta: Meta, isRemote: boolean): S3Client {
const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;
const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
const objectStorageEndpoint = useObjectStorageRemote ? meta.objectStorageRemoteEndpoint : meta.objectStorageEndpoint;
const objectStorageUseSSL = useObjectStorageRemote ? meta.objectStorageRemoteUseSSL : meta.objectStorageUseSSL;
const objectStorageUseProxy = useObjectStorageRemote ? meta.objectStorageRemoteUseProxy : meta.objectStorageUseProxy;
const objectStorageAccessKey = useObjectStorageRemote ? meta.objectStorageRemoteAccessKey : meta.objectStorageAccessKey;
const objectStorageSecretKey = useObjectStorageRemote ? meta.objectStorageRemoteSecretKey : meta.objectStorageSecretKey;
const objectStorageRegion = useObjectStorageRemote ? meta.objectStorageRemoteRegion : meta.objectStorageRegion;
const objectStorageS3ForcePathStyle = useObjectStorageRemote ? meta.objectStorageRemoteS3ForcePathStyle : meta.objectStorageS3ForcePathStyle;
const u = objectStorageEndpoint
? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}`
: `${objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
const agent = this.httpRequestService.getAgentByUrl(new URL(u), !objectStorageUseProxy);
const handlerOption: NodeHttpHandlerOptions = {};
if (meta.objectStorageUseSSL) {
if (objectStorageUseSSL) {
handlerOption.httpsAgent = agent as https.Agent;
} else {
handlerOption.httpAgent = agent as http.Agent;
}
return new S3Client({
endpoint: meta.objectStorageEndpoint ? u : undefined,
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
accessKeyId: meta.objectStorageAccessKey,
secretAccessKey: meta.objectStorageSecretKey,
endpoint: objectStorageEndpoint ? u : undefined,
credentials: (objectStorageAccessKey !== null && objectStorageSecretKey !== null) ? {
accessKeyId: objectStorageAccessKey,
secretAccessKey: objectStorageSecretKey,
} : undefined,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない
tls: meta.objectStorageUseSSL,
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
region: objectStorageRegion ? objectStorageRegion : undefined, // empty string is converted to undefined
tls: objectStorageUseSSL,
forcePathStyle: objectStorageEndpoint ? objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
requestHandler: new NodeHttpHandler(handlerOption),
});
}
@bindThis
public async upload(meta: Meta, input: PutObjectCommandInput) {
const client = this.getS3Client(meta);
public async upload(meta: Meta, input: PutObjectCommandInput, isRemote: boolean) {
const client = this.getS3Client(meta, isRemote);
return new Upload({
client,
params: input,
@ -62,8 +72,8 @@ export class S3Service {
}
@bindThis
public delete(meta: Meta, input: DeleteObjectCommandInput) {
const client = this.getS3Client(meta);
public delete(meta: Meta, input: DeleteObjectCommandInput, isRemote: boolean) {
const client = this.getS3Client(meta, isRemote);
return client.send(new DeleteObjectCommand(input));
}
}

72
packages/backend/src/models/entities/Meta.ts
View file

@ -433,6 +433,78 @@ export class Meta {
})
public objectStorageS3ForcePathStyle: boolean;
@Column('boolean', {
default: false,
})
public useObjectStorageRemote: boolean;
@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteBucket: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemotePrefix: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteBaseUrl: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteEndpoint: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteRegion: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteAccessKey: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteSecretKey: string | null;
@Column('integer', {
nullable: true,
})
public objectStorageRemotePort: number | null;
@Column('boolean', {
default: true,
})
public objectStorageRemoteUseSSL: boolean;
@Column('boolean', {
default: true,
})
public objectStorageRemoteUseProxy: boolean;
@Column('boolean', {
default: false,
})
public objectStorageRemoteSetPublicRead: boolean;
@Column('boolean', {
default: true,
})
public objectStorageRemoteS3ForcePathStyle: boolean;
@Column('boolean', {
default: false,
})

2
packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View file

@ -53,7 +53,7 @@ export class CleanRemoteFilesProcessorService {
cursor = files.at(-1)?.id ?? null;
await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true)));
await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true, true)));
deletedCount += 8;

5
packages/backend/src/queue/processors/DeleteAccountProcessorService.ts
View file

@ -10,6 +10,7 @@ import type { Note } from '@/models/entities/Note.js';
import { EmailService } from '@/core/EmailService.js';
import { bindThis } from '@/decorators.js';
import { SearchService } from '@/core/SearchService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserDeleteJobData } from '../types.js';
@ -34,6 +35,7 @@ export class DeleteAccountProcessorService {
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private userEntityService: UserEntityService,
private driveService: DriveService,
private emailService: EmailService,
private queueLoggerService: QueueLoggerService,
@ -47,6 +49,7 @@ export class DeleteAccountProcessorService {
this.logger.info(`Deleting account of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
const isRemote = user ? this.userEntityService.isRemoteUser(user) : false;
if (user == null) {
return;
}
@ -104,7 +107,7 @@ export class DeleteAccountProcessorService {
cursor = files.at(-1)?.id ?? null;
for (const file of files) {
await this.driveService.deleteFileSync(file);
await this.driveService.deleteFileSync(file, false, isRemote);
}
}

5
packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts
View file

@ -6,6 +6,7 @@ import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { bindThis } from '@/decorators.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@ -25,6 +26,7 @@ export class DeleteDriveFilesProcessorService {
private driveFilesRepository: DriveFilesRepository,
private driveService: DriveService,
private userEntityService: UserEntityService,
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('delete-drive-files');
@ -62,7 +64,8 @@ export class DeleteDriveFilesProcessorService {
cursor = files.at(-1)?.id ?? null;
for (const file of files) {
await this.driveService.deleteFileSync(file);
const isRemote = file.user ? this.userEntityService.isRemoteUser(file.user) : false;
await this.driveService.deleteFileSync(file, false, isRemote);
deletedCount++;
}

15
packages/backend/src/queue/processors/DeleteFileProcessorService.ts
View file

@ -3,6 +3,8 @@ import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import type { DriveFilesRepository } from '@/models/index.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
@ -16,7 +18,11 @@ export class DeleteFileProcessorService {
@Inject(DI.config)
private config: Config,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private driveService: DriveService,
private userEntityService: UserEntityService,
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('delete-file');
@ -26,7 +32,14 @@ export class DeleteFileProcessorService {
public async process(job: Bull.Job<ObjectStorageFileJobData>): Promise<string> {
const key: string = job.data.key;
await this.driveService.deleteObjectStorageFile(key);
const file = await this.driveFilesRepository.createQueryBuilder('file')
.where('file.accessKey = :accessKey', { accessKey: key })
.orWhere('file.thumbnailAccessKey = :thumbnailAccessKey', { thumbnailAccessKey: key })
.orWhere('file.webpublicAccessKey = :webpublicAccessKey', { webpublicAccessKey: key })
.getOne();
const isRemote = file?.user ? this.userEntityService.isRemoteUser(file.user) : false;
await this.driveService.deleteObjectStorageFile(key, isRemote);
return 'Success';
}

61
packages/backend/src/server/api/endpoints/admin/meta.ts
View file

@ -253,6 +253,54 @@ export const meta = {
type: 'boolean',
optional: true, nullable: false,
},
useObjectStorageRemote: {
type: 'boolean',
optional: true, nullable: false,
},
objectStorageRemoteBaseUrl: {
type: 'string',
optional: true, nullable: true,
},
objectStorageRemoteBucket: {
type: 'string',
optional: true, nullable: true,
},
objectStorageRemotePrefix: {
type: 'string',
optional: true, nullable: true,
},
objectStorageRemoteEndpoint: {
type: 'string',
optional: true, nullable: true,
},
objectStorageRemoteRegion: {
type: 'string',
optional: true, nullable: true,
},
objectStorageRemotePort: {
type: 'number',
optional: true, nullable: true,
},
objectStorageRemoteAccessKey: {
type: 'string',
optional: true, nullable: true,
},
objectStorageRemoteSecretKey: {
type: 'string',
optional: true, nullable: true,
},
objectStorageRemoteUseSSL: {
type: 'boolean',
optional: true, nullable: false,
},
objectStorageRemoteUseProxy: {
type: 'boolean',
optional: true, nullable: false,
},
objectStorageRemoteSetPublicRead: {
type: 'boolean',
optional: true, nullable: false,
},
enableIpLogging: {
type: 'boolean',
optional: true, nullable: false,
@ -378,6 +426,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
objectStorageUseProxy: instance.objectStorageUseProxy,
objectStorageSetPublicRead: instance.objectStorageSetPublicRead,
objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle,
useObjectStorageRemote: instance.useObjectStorageRemote,
objectStorageRemoteBaseUrl: instance.objectStorageRemoteBaseUrl,
objectStorageRemoteBucket: instance.objectStorageRemoteBucket,
objectStorageRemotePrefix: instance.objectStorageRemotePrefix,
objectStorageRemoteEndpoint: instance.objectStorageRemoteEndpoint,
objectStorageRemoteRegion: instance.objectStorageRemoteRegion,
objectStorageRemotePort: instance.objectStorageRemotePort,
objectStorageRemoteAccessKey: instance.objectStorageRemoteAccessKey,
objectStorageRemoteSecretKey: instance.objectStorageRemoteSecretKey,
objectStorageRemoteUseSSL: instance.objectStorageRemoteUseSSL,
objectStorageRemoteUseProxy: instance.objectStorageRemoteUseProxy,
objectStorageRemoteSetPublicRead: instance.objectStorageRemoteSetPublicRead,
objectStorageRemoteS3ForcePathStyle: instance.objectStorageRemoteS3ForcePathStyle,
deeplAuthKey: instance.deeplAuthKey,
deeplIsPro: instance.deeplIsPro,
ctav3SaKey: instance.ctav3SaKey,

65
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View file

@ -99,6 +99,19 @@ export const paramDef = {
objectStorageUseProxy: { type: 'boolean' },
objectStorageSetPublicRead: { type: 'boolean' },
objectStorageS3ForcePathStyle: { type: 'boolean' },
useObjectStorageRemote: { type: 'boolean' },
objectStorageRemoteBaseUrl: { type: 'string', nullable: true },
objectStorageRemoteBucket: { type: 'string', nullable: true },
objectStorageRemotePrefix: { type: 'string', nullable: true },
objectStorageRemoteEndpoint: { type: 'string', nullable: true },
objectStorageRemoteRegion: { type: 'string', nullable: true },
objectStorageRemotePort: { type: 'integer', nullable: true },
objectStorageRemoteAccessKey: { type: 'string', nullable: true },
objectStorageRemoteSecretKey: { type: 'string', nullable: true },
objectStorageRemoteUseSSL: { type: 'boolean' },
objectStorageRemoteUseProxy: { type: 'boolean' },
objectStorageRemoteSetPublicRead: { type: 'boolean' },
objectStorageRemoteS3ForcePathStyle: { type: 'boolean' },
enableIpLogging: { type: 'boolean' },
enableActiveEmailValidation: { type: 'boolean' },
enableChartsForRemoteUser: { type: 'boolean' },
@ -384,6 +397,58 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
set.objectStorageS3ForcePathStyle = ps.objectStorageS3ForcePathStyle;
}
if (ps.useObjectStorageRemote !== undefined) {
set.useObjectStorageRemote = ps.useObjectStorageRemote;
}
if (ps.objectStorageRemoteBaseUrl !== undefined) {
set.objectStorageRemoteBaseUrl = ps.objectStorageRemoteBaseUrl;
}
if (ps.objectStorageRemoteBucket !== undefined) {
set.objectStorageRemoteBucket = ps.objectStorageRemoteBucket;
}
if (ps.objectStorageRemotePrefix !== undefined) {
set.objectStorageRemotePrefix = ps.objectStorageRemotePrefix;
}
if (ps.objectStorageRemoteEndpoint !== undefined) {
set.objectStorageRemoteEndpoint = ps.objectStorageRemoteEndpoint;
}
if (ps.objectStorageRemoteRegion !== undefined) {
set.objectStorageRemoteRegion = ps.objectStorageRemoteRegion;
}
if (ps.objectStorageRemotePort !== undefined) {
set.objectStorageRemotePort = ps.objectStorageRemotePort;
}
if (ps.objectStorageRemoteAccessKey !== undefined) {
set.objectStorageRemoteAccessKey = ps.objectStorageRemoteAccessKey;
}
if (ps.objectStorageRemoteSecretKey !== undefined) {
set.objectStorageRemoteSecretKey = ps.objectStorageRemoteSecretKey;
}
if (ps.objectStorageRemoteUseSSL !== undefined) {
set.objectStorageRemoteUseSSL = ps.objectStorageRemoteUseSSL;
}
if (ps.objectStorageRemoteUseProxy !== undefined) {
set.objectStorageRemoteUseProxy = ps.objectStorageRemoteUseProxy;
}
if (ps.objectStorageRemoteSetPublicRead !== undefined) {
set.objectStorageRemoteSetPublicRead = ps.objectStorageRemoteSetPublicRead;
}
if (ps.objectStorageRemoteS3ForcePathStyle !== undefined) {
set.objectStorageRemoteS3ForcePathStyle = ps.objectStorageRemoteS3ForcePathStyle;
}
if (ps.translatorType !== undefined) {
if (ps.translatorType === '') {
set.translatorType = null;

211
packages/frontend/src/pages/admin/object-storage.vue
View file

@ -4,66 +4,139 @@
<MkSpacer :contentMax="700" :marginMin="16" :marginMax="32">
<FormSuspense :p="init">
<div class="_gaps_m">
<MkSwitch v-model="useObjectStorage">{{ i18n.ts.useObjectStorage }}</MkSwitch>
<MkFolder>
<template #label>{{ i18n.ts.objectStorage }}</template>
<template v-if="useObjectStorage">
<MkInput v-model="objectStorageBaseUrl" :placeholder="'https://example.com'">
<template #label>{{ i18n.ts.objectStorageBaseUrl }}</template>
<template #caption>{{ i18n.ts.objectStorageBaseUrlDesc }}</template>
</MkInput>
<div class="_gaps_m">
<MkSwitch v-model="useObjectStorage">{{ i18n.ts.useObjectStorage }}</MkSwitch>
<MkInput v-model="objectStorageBucket">
<template #label>{{ i18n.ts.objectStorageBucket }}</template>
<template #caption>{{ i18n.ts.objectStorageBucketDesc }}</template>
</MkInput>
<template v-if="useObjectStorage">
<MkInput v-model="objectStorageBaseUrl" :placeholder="'https://example.com'">
<template #label>{{ i18n.ts.objectStorageBaseUrl }}</template>
<template #caption>{{ i18n.ts.objectStorageBaseUrlDesc }}</template>
</MkInput>
<MkInput v-model="objectStoragePrefix">
<template #label>{{ i18n.ts.objectStoragePrefix }}</template>
<template #caption>{{ i18n.ts.objectStoragePrefixDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageBucket">
<template #label>{{ i18n.ts.objectStorageBucket }}</template>
<template #caption>{{ i18n.ts.objectStorageBucketDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageEndpoint" :placeholder="'example.com'">
<template #label>{{ i18n.ts.objectStorageEndpoint }}</template>
<template #prefix>https://</template>
<template #caption>{{ i18n.ts.objectStorageEndpointDesc }}</template>
</MkInput>
<MkInput v-model="objectStoragePrefix">
<template #label>{{ i18n.ts.objectStoragePrefix }}</template>
<template #caption>{{ i18n.ts.objectStoragePrefixDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageRegion">
<template #label>{{ i18n.ts.objectStorageRegion }}</template>
<template #caption>{{ i18n.ts.objectStorageRegionDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageEndpoint" :placeholder="'example.com'">
<template #label>{{ i18n.ts.objectStorageEndpoint }}</template>
<template #prefix>https://</template>
<template #caption>{{ i18n.ts.objectStorageEndpointDesc }}</template>
</MkInput>
<FormSplit :minWidth="280">
<MkInput v-model="objectStorageAccessKey">
<template #prefix><i class="ti ti-key"></i></template>
<template #label>Access key</template>
</MkInput>
<MkInput v-model="objectStorageRegion">
<template #label>{{ i18n.ts.objectStorageRegion }}</template>
<template #caption>{{ i18n.ts.objectStorageRegionDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageSecretKey" type="password">
<template #prefix><i class="ti ti-key"></i></template>
<template #label>Secret key</template>
</MkInput>
</FormSplit>
<FormSplit :minWidth="280">
<MkInput v-model="objectStorageAccessKey">
<template #prefix><i class="ti ti-key"></i></template>
<template #label>Access key</template>
</MkInput>
<MkSwitch v-model="objectStorageUseSSL">
<template #label>{{ i18n.ts.objectStorageUseSSL }}</template>
<template #caption>{{ i18n.ts.objectStorageUseSSLDesc }}</template>
</MkSwitch>
<MkInput v-model="objectStorageSecretKey" type="password">
<template #prefix><i class="ti ti-key"></i></template>
<template #label>Secret key</template>
</MkInput>
</FormSplit>
<MkSwitch v-model="objectStorageUseProxy">
<template #label>{{ i18n.ts.objectStorageUseProxy }}</template>
<template #caption>{{ i18n.ts.objectStorageUseProxyDesc }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageUseSSL">
<template #label>{{ i18n.ts.objectStorageUseSSL }}</template>
<template #caption>{{ i18n.ts.objectStorageUseSSLDesc }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageSetPublicRead">
<template #label>{{ i18n.ts.objectStorageSetPublicRead }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageUseProxy">
<template #label>{{ i18n.ts.objectStorageUseProxy }}</template>
<template #caption>{{ i18n.ts.objectStorageUseProxyDesc }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageS3ForcePathStyle">
<template #label>s3ForcePathStyle</template>
<template #caption>{{ i18n.ts.s3ForcePathStyleDesc }}</template>
</MkSwitch>
</template>
<MkSwitch v-model="objectStorageSetPublicRead">
<template #label>{{ i18n.ts.objectStorageSetPublicRead }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageS3ForcePathStyle">
<template #label>s3ForcePathStyle</template>
<template #caption>{{ i18n.ts.s3ForcePathStyleDesc }}</template>
</MkSwitch>
</template>
</div>
</MkFolder>
<MkFolder>
<template #label>{{ i18n.ts.objectStorage }} (Remote)</template>
<div class="_gaps_m">
<MkSwitch v-model="useObjectStorageRemote">{{ i18n.ts.useObjectStorage }}</MkSwitch>
<template v-if="useObjectStorageRemote">
<MkInput v-model="objectStorageRemoteBaseUrl" :placeholder="'https://example.com'">
<template #label>{{ i18n.ts.objectStorageBaseUrl }}</template>
<template #caption>{{ i18n.ts.objectStorageBaseUrlDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageRemoteBucket">
<template #label>{{ i18n.ts.objectStorageBucket }}</template>
<template #caption>{{ i18n.ts.objectStorageBucketDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageRemotePrefix">
<template #label>{{ i18n.ts.objectStoragePrefix }}</template>
<template #caption>{{ i18n.ts.objectStoragePrefixDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageRemoteEndpoint" :placeholder="'example.com'">
<template #label>{{ i18n.ts.objectStorageEndpoint }}</template>
<template #prefix>https://</template>
<template #caption>{{ i18n.ts.objectStorageEndpointDesc }}</template>
</MkInput>
<MkInput v-model="objectStorageRemoteRegion">
<template #label>{{ i18n.ts.objectStorageRegion }}</template>
<template #caption>{{ i18n.ts.objectStorageRegionDesc }}</template>
</MkInput>
<FormSplit :minWidth="280">
<MkInput v-model="objectStorageRemoteAccessKey">
<template #prefix><i class="ti ti-key"></i></template>
<template #label>Access key</template>
</MkInput>
<MkInput v-model="objectStorageRemoteSecretKey" type="password">
<template #prefix><i class="ti ti-key"></i></template>
<template #label>Secret key</template>
</MkInput>
</FormSplit>
<MkSwitch v-model="objectStorageRemoteUseSSL">
<template #label>{{ i18n.ts.objectStorageUseSSL }}</template>
<template #caption>{{ i18n.ts.objectStorageUseSSLDesc }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageRemoteUseProxy">
<template #label>{{ i18n.ts.objectStorageUseProxy }}</template>
<template #caption>{{ i18n.ts.objectStorageUseProxyDesc }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageRemoteSetPublicRead">
<template #label>{{ i18n.ts.objectStorageSetPublicRead }}</template>
</MkSwitch>
<MkSwitch v-model="objectStorageRemoteS3ForcePathStyle">
<template #label>s3ForcePathStyle</template>
<template #caption>{{ i18n.ts.s3ForcePathStyleDesc }}</template>
</MkSwitch>
</template>
</div>
</MkFolder>
</div>
</FormSuspense>
</MkSpacer>
@ -80,6 +153,7 @@
<script lang="ts" setup>
import { } from 'vue';
import XHeader from './_header_.vue';
import MkFolder from '@/components/MkFolder.vue';
import MkSwitch from '@/components/MkSwitch.vue';
import MkInput from '@/components/MkInput.vue';
import FormSuspense from '@/components/form/suspense.vue';
@ -103,6 +177,19 @@ let objectStorageUseSSL: boolean = $ref(false);
let objectStorageUseProxy: boolean = $ref(false);
let objectStorageSetPublicRead: boolean = $ref(false);
let objectStorageS3ForcePathStyle: boolean = $ref(true);
let useObjectStorageRemote: boolean = $ref(false);
let objectStorageRemoteBaseUrl: string | null = $ref(null);
let objectStorageRemoteBucket: string | null = $ref(null);
let objectStorageRemotePrefix: string | null = $ref(null);
let objectStorageRemoteEndpoint: string | null = $ref(null);
let objectStorageRemoteRegion: string | null = $ref(null);
let objectStorageRemotePort: number | null = $ref(null);
let objectStorageRemoteAccessKey: string | null = $ref(null);
let objectStorageRemoteSecretKey: string | null = $ref(null);
let objectStorageRemoteUseSSL: boolean = $ref(false);
let objectStorageRemoteUseProxy: boolean = $ref(false);
let objectStorageRemoteSetPublicRead: boolean = $ref(false);
let objectStorageRemoteS3ForcePathStyle: boolean = $ref(true);
async function init() {
const meta = await os.api('admin/meta');
@ -119,6 +206,19 @@ async function init() {
objectStorageUseProxy = meta.objectStorageUseProxy;
objectStorageSetPublicRead = meta.objectStorageSetPublicRead;
objectStorageS3ForcePathStyle = meta.objectStorageS3ForcePathStyle;
useObjectStorageRemote = meta.useObjectStorageRemote;
objectStorageRemoteBaseUrl = meta.objectStorageRemoteBaseUrl;
objectStorageRemoteBucket = meta.objectStorageRemoteBucket;
objectStorageRemotePrefix = meta.objectStorageRemotePrefix;
objectStorageRemoteEndpoint = meta.objectStorageRemoteEndpoint;
objectStorageRemoteRegion = meta.objectStorageRemoteRegion;
objectStorageRemotePort = meta.objectStorageRemotePort;
objectStorageRemoteAccessKey = meta.objectStorageRemoteAccessKey;
objectStorageRemoteSecretKey = meta.objectStorageRemoteSecretKey;
objectStorageRemoteUseSSL = meta.objectStorageRemoteUseSSL;
objectStorageRemoteUseProxy = meta.objectStorageRemoteUseProxy;
objectStorageRemoteSetPublicRead = meta.objectStorageRemoteSetPublicRead;
objectStorageRemoteS3ForcePathStyle = meta.objectStorageRemoteS3ForcePathStyle;
}
function save() {
@ -136,6 +236,19 @@ function save() {
objectStorageUseProxy,
objectStorageSetPublicRead,
objectStorageS3ForcePathStyle,
useObjectStorageRemote,
objectStorageRemoteBaseUrl,
objectStorageRemoteBucket,
objectStorageRemotePrefix,
objectStorageRemoteEndpoint,
objectStorageRemoteRegion,
objectStorageRemotePort,
objectStorageRemoteAccessKey,
objectStorageRemoteSecretKey,
objectStorageRemoteUseSSL,
objectStorageRemoteUseProxy,
objectStorageRemoteSetPublicRead,
objectStorageRemoteS3ForcePathStyle,
}).then(() => {
fetchInstance();
});