592 lines
20 KiB
TypeScript
592 lines
20 KiB
TypeScript
import type { Cloudly } from '../classes.cloudly.js';
|
|
import * as plugins from '../plugins.js';
|
|
import { BackupRecord } from './classes.backuprecord.js';
|
|
import { createBackupTargetWriterFromEnv, type IBackupTargetWriter } from './classes.replicationtarget.js';
|
|
|
|
export type TBackupStatus =
|
|
| 'pending'
|
|
| 'running'
|
|
| 'replicating'
|
|
| 'replicated'
|
|
| 'ready'
|
|
| 'failed'
|
|
| 'restoring'
|
|
| 'restored';
|
|
export type TBackupResourceType = 'volume' | 'database' | 'objectstorage';
|
|
export type TBackupReplicationTargetType = 's3' | 'smb';
|
|
|
|
export interface IBackupArchiveObject {
|
|
path: string;
|
|
size: number;
|
|
sha256: string;
|
|
}
|
|
|
|
export interface IBackupArchiveManifest {
|
|
version: 1;
|
|
backupId: string;
|
|
createdAt: number;
|
|
objects: IBackupArchiveObject[];
|
|
totalSize: number;
|
|
}
|
|
|
|
export interface IBackupReplicationResult {
|
|
targetType: TBackupReplicationTargetType;
|
|
targetPath: string;
|
|
manifestPath: string;
|
|
manifestSha256: string;
|
|
objectCount: number;
|
|
totalSize: number;
|
|
completedAt: number;
|
|
}
|
|
|
|
export interface IBackupSnapshotData {
|
|
type: TBackupResourceType;
|
|
snapshotId: string;
|
|
snapshotName?: string;
|
|
originalSize: number;
|
|
storedSize: number;
|
|
createdAt: number;
|
|
tags?: Record<string, string>;
|
|
volumeName?: string;
|
|
mountPath?: string;
|
|
resourceName?: string;
|
|
databaseName?: string;
|
|
bucketName?: string;
|
|
}
|
|
|
|
export interface IBackupRecordData {
|
|
id: string;
|
|
serviceId: string;
|
|
serviceName?: string;
|
|
clusterId?: string;
|
|
status: TBackupStatus;
|
|
trigger: 'manual' | 'scheduled';
|
|
snapshots: IBackupSnapshotData[];
|
|
replication?: IBackupReplicationResult;
|
|
createdAt: number;
|
|
updatedAt: number;
|
|
completedAt?: number;
|
|
requestedBy?: string;
|
|
errorText?: string;
|
|
restoreHistory?: Array<{
|
|
restoredAt: number;
|
|
status: 'restored' | 'failed';
|
|
errorText?: string;
|
|
}>;
|
|
tags?: Record<string, string>;
|
|
}
|
|
|
|
export class CloudlyBackupManager {
|
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
|
public cloudlyRef: Cloudly;
|
|
private backupTargetWriter?: IBackupTargetWriter;
|
|
|
|
get db() {
|
|
return this.cloudlyRef.mongodbConnector.smartdataDb;
|
|
}
|
|
|
|
public CBackupRecord = plugins.smartdata.setDefaultManagerForDoc(this, BackupRecord);
|
|
|
|
constructor(cloudlyRefArg: Cloudly) {
|
|
this.cloudlyRef = cloudlyRefArg;
|
|
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('createServiceBackup', async (requestArg) => {
|
|
await this.passAdminIdentity(requestArg);
|
|
return {
|
|
backup: await this.createServiceBackup(requestArg),
|
|
};
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('getServiceBackups', async (requestArg) => {
|
|
await this.passValidIdentity(requestArg);
|
|
return {
|
|
backups: await this.getBackups({
|
|
...(requestArg.serviceId ? { serviceId: requestArg.serviceId } : {}),
|
|
...(requestArg.status ? { status: requestArg.status } : {}),
|
|
}),
|
|
};
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('getBackupById', async (requestArg) => {
|
|
await this.passValidIdentity(requestArg);
|
|
return {
|
|
backup: await this.getBackupById(requestArg.backupId),
|
|
};
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('restoreServiceBackup', async (requestArg) => {
|
|
await this.passAdminIdentity(requestArg);
|
|
return {
|
|
backup: await this.restoreServiceBackup(requestArg),
|
|
};
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('prepareBackupReplication', async (requestArg) => {
|
|
await this.passClusterIdentity(requestArg);
|
|
return await this.prepareBackupReplication(requestArg);
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('uploadBackupArchiveObject', async (requestArg) => {
|
|
await this.passClusterIdentity(requestArg);
|
|
return await this.uploadBackupArchiveObject(requestArg);
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('completeBackupReplication', async (requestArg) => {
|
|
await this.passClusterIdentity(requestArg);
|
|
return await this.completeBackupReplication(requestArg);
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('getBackupArchiveManifest', async (requestArg) => {
|
|
await this.passClusterIdentity(requestArg);
|
|
return await this.getBackupArchiveManifest(requestArg);
|
|
}),
|
|
);
|
|
|
|
this.typedrouter.addTypedHandler(
|
|
new plugins.typedrequest.TypedHandler<any>('downloadBackupArchiveObject', async (requestArg) => {
|
|
await this.passClusterIdentity(requestArg);
|
|
return await this.downloadBackupArchiveObject(requestArg);
|
|
}),
|
|
);
|
|
}
|
|
|
|
public async start() {
|
|
const schedule = process.env.CLOUDLY_BACKUP_CRON;
|
|
this.cloudlyRef.taskManager.registerTask(
|
|
'backup-all-services',
|
|
new plugins.taskbuffer.Task({
|
|
name: 'backup-all-services',
|
|
taskFunction: async () => await this.backupAllServices(),
|
|
}),
|
|
{
|
|
description: 'Create backups for every workload service with backup-enabled resources.',
|
|
category: 'backup',
|
|
schedule,
|
|
enabled: Boolean(schedule),
|
|
},
|
|
);
|
|
}
|
|
|
|
public async stop() {}
|
|
|
|
public async getBackups(queryArg: Record<string, unknown> = {}) {
|
|
const backups = await this.CBackupRecord.getInstances(queryArg);
|
|
return await Promise.all(backups.map((backupArg) => backupArg.createSavableObject()));
|
|
}
|
|
|
|
public async getBackupById(backupIdArg: string) {
|
|
const backup = await BackupRecord.getInstance({ id: backupIdArg });
|
|
if (!backup) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} not found`);
|
|
}
|
|
return await backup.createSavableObject();
|
|
}
|
|
|
|
public async backupAllServices() {
|
|
const services = await this.cloudlyRef.serviceManager.CService.getInstances({});
|
|
const results: Array<{ serviceId: string; backupId?: string; errorText?: string }> = [];
|
|
for (const service of services) {
|
|
if (service.data.serviceCategory && service.data.serviceCategory !== 'workload') {
|
|
continue;
|
|
}
|
|
try {
|
|
const backup = await this.createServiceBackup({
|
|
identity: {
|
|
name: 'cloudly-backup-scheduler',
|
|
role: 'admin',
|
|
type: 'machine',
|
|
userId: 'system',
|
|
expiresAt: Date.now() + 3600 * 1000,
|
|
jwt: '',
|
|
},
|
|
serviceId: service.id,
|
|
tags: {
|
|
trigger: 'scheduled',
|
|
},
|
|
});
|
|
results.push({ serviceId: service.id, backupId: backup.id });
|
|
} catch (error) {
|
|
results.push({ serviceId: service.id, errorText: (error as Error).message });
|
|
}
|
|
}
|
|
return { results };
|
|
}
|
|
|
|
public async createServiceBackup(requestArg: {
|
|
identity: plugins.servezoneInterfaces.data.IIdentity;
|
|
serviceId: string;
|
|
clusterId?: string;
|
|
tags?: Record<string, string>;
|
|
}) {
|
|
const service = await this.cloudlyRef.serviceManager.CService.getInstance({
|
|
id: requestArg.serviceId,
|
|
});
|
|
if (!service) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Service ${requestArg.serviceId} not found`);
|
|
}
|
|
|
|
const now = Date.now();
|
|
const backup = new BackupRecord();
|
|
backup.id = await BackupRecord.getNewId();
|
|
backup.serviceId = service.id;
|
|
backup.serviceName = service.data.name;
|
|
backup.clusterId = requestArg.clusterId || (requestArg.identity as any).clusterId;
|
|
backup.status = 'running';
|
|
backup.trigger = 'manual';
|
|
backup.snapshots = [];
|
|
backup.createdAt = now;
|
|
backup.updatedAt = now;
|
|
backup.requestedBy = requestArg.identity.userId;
|
|
backup.tags = requestArg.tags;
|
|
await backup.save();
|
|
|
|
try {
|
|
const result = await this.fireCoreflowRequest('executeServiceBackup', {
|
|
backupId: backup.id,
|
|
service: await service.createSavableObject(),
|
|
tags: requestArg.tags,
|
|
replication: {
|
|
enabled: true,
|
|
},
|
|
}, backup.clusterId);
|
|
backup.snapshots = result.snapshots || [];
|
|
if (!result.replication) {
|
|
throw new Error('Coreflow did not complete remote backup replication');
|
|
}
|
|
backup.replication = result.replication;
|
|
backup.status = 'replicated';
|
|
backup.completedAt = Date.now();
|
|
backup.updatedAt = Date.now();
|
|
await backup.save();
|
|
await this.applyRetention(backup.serviceId);
|
|
} catch (error) {
|
|
backup.status = 'failed';
|
|
backup.errorText = (error as Error).message;
|
|
backup.completedAt = Date.now();
|
|
backup.updatedAt = Date.now();
|
|
await backup.save();
|
|
throw error;
|
|
}
|
|
|
|
return await backup.createSavableObject();
|
|
}
|
|
|
|
private async applyRetention(serviceIdArg: string) {
|
|
const keepLast = Number(process.env.CLOUDLY_BACKUP_KEEP_LAST || '24');
|
|
if (!Number.isInteger(keepLast) || keepLast <= 0) {
|
|
return;
|
|
}
|
|
const backups = await this.CBackupRecord.getInstances({
|
|
serviceId: serviceIdArg,
|
|
});
|
|
const completedBackups = backups
|
|
.filter((backupArg) => backupArg.status === 'replicated' || backupArg.status === 'restored' || backupArg.status === 'failed')
|
|
.sort((a, b) => (b.createdAt || 0) - (a.createdAt || 0));
|
|
for (const backup of completedBackups.slice(keepLast)) {
|
|
await backup.delete();
|
|
}
|
|
}
|
|
|
|
public async restoreServiceBackup(requestArg: {
|
|
identity: plugins.servezoneInterfaces.data.IIdentity;
|
|
backupId: string;
|
|
clear?: boolean;
|
|
resourceTypes?: TBackupResourceType[];
|
|
}) {
|
|
const backup = await BackupRecord.getInstance({ id: requestArg.backupId });
|
|
if (!backup) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Backup ${requestArg.backupId} not found`);
|
|
}
|
|
if (backup.status !== 'replicated' && backup.status !== 'restored') {
|
|
throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} is not restorable in status ${backup.status}`);
|
|
}
|
|
const service = await this.cloudlyRef.serviceManager.CService.getInstance({
|
|
id: backup.serviceId,
|
|
});
|
|
if (!service) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Service ${backup.serviceId} not found`);
|
|
}
|
|
|
|
const previousStatus = backup.status;
|
|
backup.status = 'restoring';
|
|
backup.updatedAt = Date.now();
|
|
await backup.save();
|
|
|
|
try {
|
|
await this.fireCoreflowRequest('executeServiceRestore', {
|
|
backupId: backup.id,
|
|
service: await service.createSavableObject(),
|
|
snapshots: backup.snapshots || [],
|
|
clear: requestArg.clear,
|
|
resourceTypes: requestArg.resourceTypes,
|
|
replication: {
|
|
enabled: true,
|
|
},
|
|
}, backup.clusterId);
|
|
backup.status = 'restored';
|
|
backup.restoreHistory = [
|
|
...(backup.restoreHistory || []),
|
|
{
|
|
restoredAt: Date.now(),
|
|
status: 'restored',
|
|
},
|
|
];
|
|
backup.updatedAt = Date.now();
|
|
await backup.save();
|
|
} catch (error) {
|
|
backup.status = previousStatus;
|
|
backup.restoreHistory = [
|
|
...(backup.restoreHistory || []),
|
|
{
|
|
restoredAt: Date.now(),
|
|
status: 'failed',
|
|
errorText: (error as Error).message,
|
|
},
|
|
];
|
|
backup.updatedAt = Date.now();
|
|
await backup.save();
|
|
throw error;
|
|
}
|
|
|
|
return await backup.createSavableObject();
|
|
}
|
|
|
|
private getBackupTargetWriter() {
|
|
if (!this.backupTargetWriter) {
|
|
this.backupTargetWriter = createBackupTargetWriterFromEnv();
|
|
}
|
|
return this.backupTargetWriter;
|
|
}
|
|
|
|
private normalizeTargetPath(pathArg: string) {
|
|
const normalized = plugins.path.posix
|
|
.normalize(String(pathArg || '').replace(/\\/g, '/').trim())
|
|
.replace(/^\/+/, '');
|
|
if (!normalized || normalized === '.' || normalized.startsWith('../') || normalized.includes('/../')) {
|
|
throw new Error(`Invalid backup target path ${pathArg}`);
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
private getBackupTargetPath(backupArg: BackupRecord) {
|
|
return this.normalizeTargetPath([
|
|
process.env.CLOUDLY_BACKUP_TARGET_PREFIX || 'serve.zone-backups',
|
|
'clusters',
|
|
backupArg.clusterId || 'default',
|
|
'services',
|
|
backupArg.serviceId,
|
|
'backups',
|
|
backupArg.id,
|
|
].filter(Boolean).join('/'));
|
|
}
|
|
|
|
private getArchiveObjectTargetPath(backupArg: BackupRecord, objectPathArg: string) {
|
|
return this.normalizeTargetPath(`${this.getBackupTargetPath(backupArg)}/archive/${objectPathArg}`);
|
|
}
|
|
|
|
private getManifestTargetPath(backupArg: BackupRecord) {
|
|
return this.normalizeTargetPath(`${this.getBackupTargetPath(backupArg)}/manifest.json`);
|
|
}
|
|
|
|
private getSha256(contentsArg: Buffer) {
|
|
return plugins.crypto.createHash('sha256').update(contentsArg).digest('hex');
|
|
}
|
|
|
|
private assertObjectMatches(objectArg: IBackupArchiveObject, contentsArg: Buffer) {
|
|
if (contentsArg.length !== objectArg.size || this.getSha256(contentsArg) !== objectArg.sha256) {
|
|
throw new Error(`Backup archive object checksum mismatch for ${objectArg.path}`);
|
|
}
|
|
}
|
|
|
|
private createManifestBuffer(backupArg: BackupRecord, manifestArg: IBackupArchiveManifest) {
|
|
return Buffer.from(`${JSON.stringify({
|
|
version: 1,
|
|
backupId: backupArg.id,
|
|
serviceId: backupArg.serviceId,
|
|
serviceName: backupArg.serviceName,
|
|
clusterId: backupArg.clusterId,
|
|
archive: manifestArg,
|
|
}, null, 2)}\n`);
|
|
}
|
|
|
|
private async getBackupForClusterRequest(backupIdArg: string, identityArg: plugins.servezoneInterfaces.data.IIdentity) {
|
|
const backup = await BackupRecord.getInstance({ id: backupIdArg });
|
|
if (!backup) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} not found`);
|
|
}
|
|
const identityClusterId = (identityArg as any).clusterId;
|
|
if (backup.clusterId && identityClusterId && backup.clusterId !== identityClusterId) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} does not belong to this cluster`);
|
|
}
|
|
return backup;
|
|
}
|
|
|
|
public async prepareBackupReplication(requestArg: {
|
|
identity: plugins.servezoneInterfaces.data.IIdentity;
|
|
backupId: string;
|
|
manifest: IBackupArchiveManifest;
|
|
}) {
|
|
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
|
|
const targetWriter = this.getBackupTargetWriter();
|
|
const missingObjects: IBackupArchiveObject[] = [];
|
|
for (const object of requestArg.manifest.objects || []) {
|
|
const targetPath = this.getArchiveObjectTargetPath(backup, object.path);
|
|
if (!await targetWriter.hasObject(targetPath, object)) {
|
|
missingObjects.push(object);
|
|
}
|
|
}
|
|
backup.status = 'replicating';
|
|
backup.updatedAt = Date.now();
|
|
await backup.save();
|
|
return { missingObjects };
|
|
}
|
|
|
|
public async uploadBackupArchiveObject(requestArg: {
|
|
identity: plugins.servezoneInterfaces.data.IIdentity;
|
|
backupId: string;
|
|
object: IBackupArchiveObject;
|
|
contentsBase64: string;
|
|
}) {
|
|
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
|
|
const contents = Buffer.from(requestArg.contentsBase64 || '', 'base64');
|
|
this.assertObjectMatches(requestArg.object, contents);
|
|
await this.getBackupTargetWriter().putObject(
|
|
this.getArchiveObjectTargetPath(backup, requestArg.object.path),
|
|
requestArg.object,
|
|
contents,
|
|
);
|
|
return { accepted: true };
|
|
}
|
|
|
|
public async completeBackupReplication(requestArg: {
|
|
identity: plugins.servezoneInterfaces.data.IIdentity;
|
|
backupId: string;
|
|
manifest: IBackupArchiveManifest;
|
|
}) {
|
|
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
|
|
const targetWriter = this.getBackupTargetWriter();
|
|
for (const object of requestArg.manifest.objects || []) {
|
|
const targetPath = this.getArchiveObjectTargetPath(backup, object.path);
|
|
if (!await targetWriter.hasObject(targetPath, object)) {
|
|
throw new Error(`Remote backup target is missing archive object ${object.path}`);
|
|
}
|
|
}
|
|
|
|
const manifestPath = this.getManifestTargetPath(backup);
|
|
const manifestBuffer = this.createManifestBuffer(backup, requestArg.manifest);
|
|
const manifestObject = {
|
|
path: 'manifest.json',
|
|
size: manifestBuffer.length,
|
|
sha256: this.getSha256(manifestBuffer),
|
|
};
|
|
await targetWriter.putObject(manifestPath, manifestObject, manifestBuffer);
|
|
|
|
const replication: IBackupReplicationResult = {
|
|
targetType: targetWriter.targetType,
|
|
targetPath: this.getBackupTargetPath(backup),
|
|
manifestPath,
|
|
manifestSha256: manifestObject.sha256,
|
|
objectCount: requestArg.manifest.objects.length,
|
|
totalSize: requestArg.manifest.totalSize,
|
|
completedAt: Date.now(),
|
|
};
|
|
backup.replication = replication;
|
|
backup.status = 'replicated';
|
|
backup.completedAt = replication.completedAt;
|
|
backup.updatedAt = replication.completedAt;
|
|
await backup.save();
|
|
return { replication };
|
|
}
|
|
|
|
public async getBackupArchiveManifest(requestArg: {
|
|
identity: plugins.servezoneInterfaces.data.IIdentity;
|
|
backupId: string;
|
|
}) {
|
|
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
|
|
if (!backup.replication) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} has not been replicated`);
|
|
}
|
|
const manifestBuffer = await this.getBackupTargetWriter().readObject(backup.replication.manifestPath);
|
|
if (this.getSha256(manifestBuffer) !== backup.replication.manifestSha256) {
|
|
throw new Error(`Remote manifest checksum mismatch for backup ${backup.id}`);
|
|
}
|
|
const parsedManifest = JSON.parse(manifestBuffer.toString('utf8'));
|
|
return { manifest: parsedManifest.archive as IBackupArchiveManifest };
|
|
}
|
|
|
|
public async downloadBackupArchiveObject(requestArg: {
|
|
identity: plugins.servezoneInterfaces.data.IIdentity;
|
|
backupId: string;
|
|
object: IBackupArchiveObject;
|
|
}) {
|
|
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
|
|
if (!backup.replication) {
|
|
throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} has not been replicated`);
|
|
}
|
|
const contents = await this.getBackupTargetWriter().readObject(
|
|
this.getArchiveObjectTargetPath(backup, requestArg.object.path),
|
|
);
|
|
this.assertObjectMatches(requestArg.object, contents);
|
|
return {
|
|
object: requestArg.object,
|
|
contentsBase64: contents.toString('base64'),
|
|
};
|
|
}
|
|
|
|
private async fireCoreflowRequest(methodArg: string, payloadArg: Record<string, unknown>, clusterIdArg?: string) {
|
|
const typedsocket = this.cloudlyRef.server.typedServer?.typedsocket;
|
|
if (!typedsocket) {
|
|
throw new Error('Cloudly TypedSocket server is not running');
|
|
}
|
|
|
|
const connections = await typedsocket.findAllTargetConnections(async (connectionArg) => {
|
|
const identityTag = await connectionArg.getTagById('identity');
|
|
const identity = identityTag?.payload as plugins.servezoneInterfaces.data.IIdentity | undefined;
|
|
return identity?.role === 'cluster' && (!clusterIdArg || (identity as any).clusterId === clusterIdArg);
|
|
});
|
|
if (connections.length === 0) {
|
|
throw new Error(clusterIdArg
|
|
? `No connected coreflow for cluster ${clusterIdArg}`
|
|
: 'No connected coreflow');
|
|
}
|
|
|
|
const request = typedsocket.createTypedRequest<any>(methodArg, connections[0]);
|
|
return await request.fire(payloadArg as any);
|
|
}
|
|
|
|
private async passValidIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
|
|
await plugins.smartguard.passGuardsOrReject(requestData, [
|
|
this.cloudlyRef.authManager.validIdentityGuard,
|
|
]);
|
|
}
|
|
|
|
private async passAdminIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
|
|
await plugins.smartguard.passGuardsOrReject(requestData, [
|
|
this.cloudlyRef.authManager.adminIdentityGuard,
|
|
]);
|
|
}
|
|
|
|
private async passClusterIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
|
|
await this.passValidIdentity(requestData);
|
|
if (requestData.identity.role !== 'cluster') {
|
|
throw new plugins.typedrequest.TypedResponseError('Cluster identity required');
|
|
}
|
|
}
|
|
}
|