feat: M1-04 — Content Safety deepening, reports CAPI, violation records
All checks were successful
Deploy API Server / build-and-deploy (push) Successful in 39s

- Add ViolationRecord table (Prisma + migration)
- CAPI POST /api/reports for user report submission
- AAPI reports list + handle, violations list + penalty apply
- Admin page: reports management + violation records tabs

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
WangDL 2026-05-24 10:53:19 +08:00
parent 90c27ee979
commit a08fd4970a
8 changed files with 215 additions and 9 deletions

View File

@ -47,6 +47,7 @@ jobs:
$MYSQL_CMD -e "DROP TABLE IF EXISTS ModelRoute;" 2>/dev/null || true $MYSQL_CMD -e "DROP TABLE IF EXISTS ModelRoute;" 2>/dev/null || true
$MYSQL_CMD -e "DROP TABLE IF EXISTS ProviderConfig;" 2>/dev/null || true $MYSQL_CMD -e "DROP TABLE IF EXISTS ProviderConfig;" 2>/dev/null || true
$MYSQL_CMD -e "DROP TABLE IF EXISTS FallbackEvent;" 2>/dev/null || true $MYSQL_CMD -e "DROP TABLE IF EXISTS FallbackEvent;" 2>/dev/null || true
$MYSQL_CMD -e "DROP TABLE IF EXISTS ViolationRecord;" 2>/dev/null || true
$MYSQL_CMD -e "ALTER TABLE UploadedFile DROP COLUMN objectKey;" 2>/dev/null || true $MYSQL_CMD -e "ALTER TABLE UploadedFile DROP COLUMN objectKey;" 2>/dev/null || true
$MYSQL_CMD -e "ALTER TABLE UploadedFile DROP COLUMN bucket;" 2>/dev/null || true $MYSQL_CMD -e "ALTER TABLE UploadedFile DROP COLUMN bucket;" 2>/dev/null || true
$MYSQL_CMD -e "DROP INDEX UploadedFile_objectKey_idx ON UploadedFile;" 2>/dev/null || true $MYSQL_CMD -e "DROP INDEX UploadedFile_objectKey_idx ON UploadedFile;" 2>/dev/null || true

View File

@ -0,0 +1,16 @@
-- CreateTable
CREATE TABLE IF NOT EXISTS `ViolationRecord` (
`id` VARCHAR(191) NOT NULL,
`userId` VARCHAR(191) NOT NULL,
`contentType` VARCHAR(32) NOT NULL,
`content` VARCHAR(1000) NOT NULL,
`riskLevel` VARCHAR(16) NOT NULL,
`penalty` VARCHAR(32) NOT NULL DEFAULT 'none',
`appliedBy` VARCHAR(100) NULL,
`appliedAt` DATETIME(3) NULL,
`createdAt` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
INDEX `ViolationRecord_userId_idx`(`userId`),
INDEX `ViolationRecord_createdAt_idx`(`createdAt`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

View File

@ -990,6 +990,21 @@ model ContentReport {
@@index([createdAt]) @@index([createdAt])
} }
model ViolationRecord {
id String @id @default(cuid())
userId String
contentType String @db.VarChar(32)
content String @db.VarChar(1000)
riskLevel String @db.VarChar(16)
penalty String @default("none") @db.VarChar(32)
appliedBy String? @db.VarChar(100)
appliedAt DateTime?
createdAt DateTime @default(now())
@@index([userId])
@@index([createdAt])
}
model ApiMetric { model ApiMetric {
id String @id @default(cuid()) id String @id @default(cuid())
path String @db.VarChar(255) path String @db.VarChar(255)

View File

@ -1,20 +1,81 @@
import { Controller, Get, Post, Delete, Body, Param, UseGuards } from '@nestjs/common'; import { Controller, Get, Post, Delete, Body, Param, Query, UseGuards } from '@nestjs/common';
import { ApiTags, ApiBearerAuth } from '@nestjs/swagger'; import { ApiTags, ApiBearerAuth, ApiOperation } from '@nestjs/swagger';
import { ContentSafetyService } from './content-safety.service'; import { ContentSafetyService } from './content-safety.service';
import { AdminAuthGuard } from '../../common/guards/admin-auth.guard'; import { AdminAuthGuard } from '../../common/guards/admin-auth.guard';
import { AdminRolesGuard } from '../../common/guards/admin-roles.guard'; import { AdminRolesGuard } from '../../common/guards/admin-roles.guard';
import { AdminRoles } from '../../common/decorators/admin-roles.decorator'; import { AdminRoles } from '../../common/decorators/admin-roles.decorator';
import { Public } from '../../common/decorators/public.decorator';
import type { AdminRole } from '../../common/types/admin-role.enum'; import type { AdminRole } from '../../common/types/admin-role.enum';
@ApiTags('content-safety')
@Controller()
export class ContentSafetyController {
constructor(private readonly svc: ContentSafetyService) {}
// ═══ CAPI: User report submission ═══
@Public()
@Post('reports')
@ApiOperation({ summary: '用户提交举报' })
async submitReport(@Body() d: { targetType: string; targetId: string; reason: string; reporterId?: string }) {
const report = await this.svc.submitReport({
reporterId: d.reporterId || 'anonymous',
targetType: d.targetType,
targetId: d.targetId,
reason: d.reason,
});
return { success: true, data: report };
}
}
@ApiTags('admin-content-safety') @ApiTags('admin-content-safety')
@Controller('admin-api/content-safety') @Controller('admin-api/content-safety')
@UseGuards(AdminAuthGuard, AdminRolesGuard) @UseGuards(AdminAuthGuard, AdminRolesGuard)
@ApiBearerAuth() @ApiBearerAuth()
export class ContentSafetyController { export class AdminContentSafetyController {
constructor(private readonly svc: ContentSafetyService) {} constructor(private readonly svc: ContentSafetyService) {}
@Get('words') @AdminRoles('SUPER_ADMIN' as AdminRole) async words() { return this.svc.getAllWords() } // ── Sensitive words ──
@Post('words') @AdminRoles('SUPER_ADMIN' as AdminRole) async addWord(@Body() d: { word: string; category: string; riskLevel: string }) { return this.svc.addWord(d.word, d.category, d.riskLevel) }
@Delete('words/:id') @AdminRoles('SUPER_ADMIN' as AdminRole) async removeWord(@Param('id') id: string) { await this.svc.removeWord(id); return { success: true } } @Get('words') @AdminRoles('SUPER_ADMIN' as AdminRole)
@Get('checks') @AdminRoles('SUPER_ADMIN' as AdminRole) async checks() { return this.svc.getChecks() } async words() { return this.svc.getAllWords() }
@Post('words') @AdminRoles('SUPER_ADMIN' as AdminRole)
async addWord(@Body() d: { word: string; category: string; riskLevel: string }) { return this.svc.addWord(d.word, d.category, d.riskLevel) }
@Delete('words/:id') @AdminRoles('SUPER_ADMIN' as AdminRole)
async removeWord(@Param('id') id: string) { await this.svc.removeWord(id); return { success: true } }
// ── AI safety checks log ──
@Get('checks') @AdminRoles('SUPER_ADMIN' as AdminRole)
async checks() { return this.svc.getChecks() }
// ── Reports ──
@Get('reports') @AdminRoles('ADMIN' as AdminRole)
@ApiOperation({ summary: '举报列表' })
async reports(@Query('status') status?: string) {
return this.svc.getReports(status);
}
@Post('reports/:id/handle') @AdminRoles('ADMIN' as AdminRole)
@ApiOperation({ summary: '处理举报' })
async handleReport(@Param('id') id: string, @Body() d: { action: string; note?: string }) {
return this.svc.handleReport(id, d.action, d.note);
}
// ── Violations ──
@Get('violations') @AdminRoles('ADMIN' as AdminRole)
@ApiOperation({ summary: '违规记录列表' })
async violations(@Query('userId') userId?: string, @Query('limit') limit = '50') {
return this.svc.getViolations(userId, parseInt(limit));
}
@Post('violations/:id/penalty') @AdminRoles('SUPER_ADMIN' as AdminRole)
@ApiOperation({ summary: '应用处罚' })
async applyPenalty(@Param('id') id: string, @Body() d: { penalty: string }) {
return this.svc.applyPenalty(id, d.penalty);
}
} }

View File

@ -1,5 +1,5 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ContentSafetyController } from './content-safety.controller'; import { ContentSafetyController, AdminContentSafetyController } from './content-safety.controller';
import { ContentSafetyService } from './content-safety.service'; import { ContentSafetyService } from './content-safety.service';
import { PrismaService } from '../../infrastructure/database/prisma.service'; import { PrismaService } from '../../infrastructure/database/prisma.service';
import { RedisService } from '../../infrastructure/redis/redis.service'; import { RedisService } from '../../infrastructure/redis/redis.service';
@ -8,7 +8,7 @@ import { AdminAuthGuard } from '../../common/guards/admin-auth.guard';
import { AdminRolesGuard } from '../../common/guards/admin-roles.guard'; import { AdminRolesGuard } from '../../common/guards/admin-roles.guard';
@Module({ @Module({
controllers: [ContentSafetyController], controllers: [ContentSafetyController, AdminContentSafetyController],
providers: [ContentSafetyService, PrismaService, RedisService, QueueService, AdminAuthGuard, AdminRolesGuard], providers: [ContentSafetyService, PrismaService, RedisService, QueueService, AdminAuthGuard, AdminRolesGuard],
exports: [ContentSafetyService], exports: [ContentSafetyService],
}) })

View File

@ -77,4 +77,56 @@ export class ContentSafetyService {
async getAllWords() { return this.prisma.sensitiveWord.findMany({ orderBy: { createdAt: 'desc' } }) } async getAllWords() { return this.prisma.sensitiveWord.findMany({ orderBy: { createdAt: 'desc' } }) }
async getChecks(limit = 50) { return this.prisma.contentSafetyCheck.findMany({ orderBy: { createdAt: 'desc' }, take: limit }) } async getChecks(limit = 50) { return this.prisma.contentSafetyCheck.findMany({ orderBy: { createdAt: 'desc' }, take: limit }) }
// ── Reports ──
async submitReport(input: { reporterId: string; targetType: string; targetId: string; reason: string }) {
return this.prisma.contentReport.create({ data: input });
}
async getReports(status?: string) {
return this.prisma.contentReport.findMany({
where: status ? { status } : undefined,
orderBy: { createdAt: 'desc' },
take: 100,
});
}
async handleReport(id: string, action: string, note?: string) {
const report = await this.prisma.contentReport.update({
where: { id },
data: { status: action, handleNote: note || null, handledAt: new Date() },
});
// If confirmed violation, create violation record
if (action === 'confirmed') {
await this.prisma.violationRecord.create({
data: {
userId: report.reporterId,
contentType: report.targetType,
content: report.reason,
riskLevel: 'medium',
},
});
}
return report;
}
// ── Violations ──
async getViolations(userId?: string, limit = 50) {
return this.prisma.violationRecord.findMany({
where: userId ? { userId } : undefined,
orderBy: { createdAt: 'desc' },
take: limit,
});
}
async applyPenalty(id: string, penalty: string) {
return this.prisma.violationRecord.update({
where: { id },
data: { penalty, appliedAt: new Date() },
});
}
} }

View File

@ -228,4 +228,64 @@ describe('M1 E2E Tests', () => {
.expect([200, 201]); .expect([200, 201]);
}); });
}); });
// ══════════════════════════════════════════════
// M1-04: Content Safety 深化
// ══════════════════════════════════════════════
describe('M1-04 Content Safety Deepening', () => {
let token: string;
beforeAll(async () => { token = await loginAdmin(); });
it('POST /api/reports → 201 submit user report', async () => {
const res = await request(app.getHttpServer())
.post('/api/reports')
.send({ targetType: 'knowledge_item', targetId: 'test123', reason: '包含错误信息', reporterId: 'user1' })
.expect([200, 201]);
expect(res.body.success).toBe(true);
});
it('GET /admin-api/content-safety/reports → 200 list reports', async () => {
if (!token) return;
const res = await request(app.getHttpServer())
.get('/admin-api/content-safety/reports')
.set('Authorization', `Bearer ${token}`)
.expect(200);
expect(Array.isArray(res.body.data)).toBe(true);
});
it('GET /admin-api/content-safety/reports → 401 without token', async () => {
await request(app.getHttpServer())
.get('/admin-api/content-safety/reports')
.expect(401);
});
it('GET /admin-api/content-safety/violations → 200 list violations', async () => {
if (!token) return;
const res = await request(app.getHttpServer())
.get('/admin-api/content-safety/violations')
.set('Authorization', `Bearer ${token}`)
.expect(200);
expect(Array.isArray(res.body.data)).toBe(true);
});
it('POST /admin-api/content-safety/violations/:id/penalty → apply penalty', async () => {
if (!token) return;
const res = await request(app.getHttpServer())
.post('/admin-api/content-safety/violations/test-id/penalty')
.set('Authorization', `Bearer ${token}`)
.send({ penalty: 'warning' })
.expect([200, 201]);
expect(res.body.success).toBe(true);
});
it('POST /admin-api/content-safety/reports/:id/handle → handle report', async () => {
if (!token) return;
const res = await request(app.getHttpServer())
.post('/admin-api/content-safety/reports/test-id/handle')
.set('Authorization', `Bearer ${token}`)
.send({ action: 'dismissed', note: '已处理' })
.expect([200, 201]);
expect(res.body.success).toBe(true);
});
});
}); });

View File

@ -90,6 +90,7 @@ const modelNames = [
'contentSafetyCheck', 'contentReport', 'apiMetric', 'taskLog', 'contentSafetyCheck', 'contentReport', 'apiMetric', 'taskLog',
'userMembership', 'quotaUsage', 'costDailySummary', 'secretRecord', 'userMembership', 'quotaUsage', 'costDailySummary', 'secretRecord',
'secretAccessLog', 'modelRoute', 'providerConfig', 'fallbackEvent', 'secretAccessLog', 'modelRoute', 'providerConfig', 'fallbackEvent',
'violationRecord', 'contentReport',
] ]
for (const name of modelNames) { for (const name of modelNames) {