perf: replace N+1 dedup queries with batch pre-fetch in convertCandidates
All checks were successful
Deploy API Server / build-and-deploy (push) Successful in 46s
All checks were successful
Deploy API Server / build-and-deploy (push) Successful in 46s
- convertQuizCandidates: batch findMany all stems before loop, dedup in-memory - convertFlashcardCandidates: batch findMany all fronts before loop, dedup in-memory - 50 items now = 1 query instead of 50 - Update tests: mock findMany instead of per-item findFirst Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
c2e5590718
commit
7725b3d2ea
@ -67,8 +67,8 @@ describe('RuntimeInternalService', () => {
|
|||||||
mockQuestionGenerationPlan = { updateMany: jest.fn() };
|
mockQuestionGenerationPlan = { updateMany: jest.fn() };
|
||||||
mockFlashcardGenerationPlan = { updateMany: jest.fn() };
|
mockFlashcardGenerationPlan = { updateMany: jest.fn() };
|
||||||
mockQuiz = { create: jest.fn(), update: jest.fn() };
|
mockQuiz = { create: jest.fn(), update: jest.fn() };
|
||||||
mockQuizQuestion = { findFirst: jest.fn(), create: jest.fn() };
|
mockQuizQuestion = { findMany: jest.fn(), create: jest.fn() };
|
||||||
mockFlashcard = { findFirst: jest.fn(), create: jest.fn() };
|
mockFlashcard = { findMany: jest.fn(), create: jest.fn() };
|
||||||
mockKnowledgeItem = { findMany: jest.fn().mockResolvedValue([]) };
|
mockKnowledgeItem = { findMany: jest.fn().mockResolvedValue([]) };
|
||||||
mockNotification = { create: jest.fn() };
|
mockNotification = { create: jest.fn() };
|
||||||
|
|
||||||
@ -558,7 +558,7 @@ describe('RuntimeInternalService', () => {
|
|||||||
mockAiRuntimeJob.update.mockResolvedValue({});
|
mockAiRuntimeJob.update.mockResolvedValue({});
|
||||||
mockQuestionGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
mockQuestionGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
||||||
mockQuiz.create.mockResolvedValue({ id: 'quiz-1' });
|
mockQuiz.create.mockResolvedValue({ id: 'quiz-1' });
|
||||||
mockQuizQuestion.findFirst.mockResolvedValue(null);
|
mockQuizQuestion.findMany.mockResolvedValue([]);
|
||||||
mockQuizQuestion.create.mockResolvedValue({});
|
mockQuizQuestion.create.mockResolvedValue({});
|
||||||
|
|
||||||
await service.submitResult('j1', qDto);
|
await service.submitResult('j1', qDto);
|
||||||
@ -576,7 +576,7 @@ describe('RuntimeInternalService', () => {
|
|||||||
mockAiRuntimeResult.create.mockResolvedValue({});
|
mockAiRuntimeResult.create.mockResolvedValue({});
|
||||||
mockAiRuntimeJob.update.mockResolvedValue({});
|
mockAiRuntimeJob.update.mockResolvedValue({});
|
||||||
mockFlashcardGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
mockFlashcardGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
||||||
mockFlashcard.findFirst.mockResolvedValue(null);
|
mockFlashcard.findMany.mockResolvedValue([]);
|
||||||
mockFlashcard.create.mockResolvedValue({});
|
mockFlashcard.create.mockResolvedValue({});
|
||||||
|
|
||||||
await service.submitResult('j1', fcDto);
|
await service.submitResult('j1', fcDto);
|
||||||
@ -860,7 +860,7 @@ describe('RuntimeInternalService', () => {
|
|||||||
mockAiRuntimeJob.update.mockResolvedValue({});
|
mockAiRuntimeJob.update.mockResolvedValue({});
|
||||||
mockQuestionGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
mockQuestionGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
||||||
mockQuiz.create.mockResolvedValue({ id: 'quiz-1' });
|
mockQuiz.create.mockResolvedValue({ id: 'quiz-1' });
|
||||||
mockQuizQuestion.findFirst.mockResolvedValue(null);
|
mockQuizQuestion.findMany.mockResolvedValue([]);
|
||||||
mockQuizQuestion.create.mockResolvedValue({});
|
mockQuizQuestion.create.mockResolvedValue({});
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -892,8 +892,8 @@ describe('RuntimeInternalService', () => {
|
|||||||
it('skips duplicate questions (same stem for user)', async () => {
|
it('skips duplicate questions (same stem for user)', async () => {
|
||||||
const loggerSpy = jest.spyOn((service as any).logger, 'warn');
|
const loggerSpy = jest.spyOn((service as any).logger, 'warn');
|
||||||
mockAiRuntimeJob.findUnique.mockResolvedValue(qJob);
|
mockAiRuntimeJob.findUnique.mockResolvedValue(qJob);
|
||||||
mockQuizQuestion.findFirst.mockResolvedValueOnce({ id: 'existing-q' }); // duplicate
|
// Batch dedup: findMany returns existing stems; 'Duplicate?' is duplicate, 'Unique?' is new
|
||||||
mockQuizQuestion.findFirst.mockResolvedValueOnce(null); // unique
|
mockQuizQuestion.findMany.mockResolvedValue([{ stem: 'Duplicate?' }]);
|
||||||
mockQuiz.update.mockResolvedValue({});
|
mockQuiz.update.mockResolvedValue({});
|
||||||
|
|
||||||
await service.submitResult('j1', {
|
await service.submitResult('j1', {
|
||||||
@ -946,7 +946,7 @@ describe('RuntimeInternalService', () => {
|
|||||||
mockAiRuntimeResult.create.mockResolvedValue({});
|
mockAiRuntimeResult.create.mockResolvedValue({});
|
||||||
mockAiRuntimeJob.update.mockResolvedValue({});
|
mockAiRuntimeJob.update.mockResolvedValue({});
|
||||||
mockFlashcardGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
mockFlashcardGenerationPlan.updateMany.mockResolvedValue({ count: 1 });
|
||||||
mockFlashcard.findFirst.mockResolvedValue(null);
|
mockFlashcard.findMany.mockResolvedValue([]);
|
||||||
mockFlashcard.create.mockResolvedValue({});
|
mockFlashcard.create.mockResolvedValue({});
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -966,8 +966,8 @@ describe('RuntimeInternalService', () => {
|
|||||||
it('skips duplicate cards (same front for user)', async () => {
|
it('skips duplicate cards (same front for user)', async () => {
|
||||||
const loggerSpy = jest.spyOn((service as any).logger, 'warn');
|
const loggerSpy = jest.spyOn((service as any).logger, 'warn');
|
||||||
mockAiRuntimeJob.findUnique.mockResolvedValue(fcJob);
|
mockAiRuntimeJob.findUnique.mockResolvedValue(fcJob);
|
||||||
mockFlashcard.findFirst.mockResolvedValueOnce({ id: 'dup-1' });
|
// Batch dedup: findMany returns existing fronts; 'Dup' is duplicate, 'Unique' is new
|
||||||
mockFlashcard.findFirst.mockResolvedValueOnce(null);
|
mockFlashcard.findMany.mockResolvedValue([{ front: 'Dup' }]);
|
||||||
|
|
||||||
await service.submitResult('j1', {
|
await service.submitResult('j1', {
|
||||||
runtimeInstanceId: 'rt-1', schemaVersion: 'ov1', status: 'succeeded',
|
runtimeInstanceId: 'rt-1', schemaVersion: 'ov1', status: 'succeeded',
|
||||||
|
|||||||
@ -458,19 +458,24 @@ export class RuntimeInternalService {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let created = 0;
|
let created = 0;
|
||||||
for (let i = 0; i < valid.length; i++) {
|
|
||||||
const q = valid[i];
|
|
||||||
const stem = q.stem ?? q.question;
|
|
||||||
|
|
||||||
// Dedup: skip if same stem already exists for this user in any quiz
|
// Batch pre-fetch existing stems for dedup (avoids N+1 per-item queries)
|
||||||
const existing = await this.prisma.quizQuestion.findFirst({
|
const stems = valid.map(q => q.stem ?? q.question);
|
||||||
|
const existingStems = new Set(
|
||||||
|
(await this.prisma.quizQuestion.findMany({
|
||||||
where: {
|
where: {
|
||||||
stem,
|
stem: { in: stems },
|
||||||
quiz: { userId: job.userId },
|
quiz: { userId: job.userId },
|
||||||
},
|
},
|
||||||
select: { id: true },
|
select: { stem: true },
|
||||||
});
|
})).map(r => r.stem),
|
||||||
if (existing) {
|
);
|
||||||
|
|
||||||
|
for (let i = 0; i < valid.length; i++) {
|
||||||
|
const q = valid[i];
|
||||||
|
const stem = stems[i];
|
||||||
|
|
||||||
|
if (existingStems.has(stem)) {
|
||||||
this.logger.warn(`convertQuizCandidates: skipping duplicate question stem="${stem.substring(0, 50)}"`);
|
this.logger.warn(`convertQuizCandidates: skipping duplicate question stem="${stem.substring(0, 50)}"`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -521,6 +526,15 @@ export class RuntimeInternalService {
|
|||||||
)
|
)
|
||||||
: new Set<string>();
|
: new Set<string>();
|
||||||
|
|
||||||
|
// Batch pre-fetch existing fronts for dedup (avoids N+1 per-item queries)
|
||||||
|
const fronts = valid.map(c => c.front ?? c.question);
|
||||||
|
const existingFronts = new Set(
|
||||||
|
(await this.prisma.flashcard.findMany({
|
||||||
|
where: { userId: job.userId, front: { in: fronts }, deletedAt: null },
|
||||||
|
select: { front: true },
|
||||||
|
})).map(r => r.front),
|
||||||
|
);
|
||||||
|
|
||||||
for (const card of valid) {
|
for (const card of valid) {
|
||||||
const rawIds: string[] = Array.isArray(card.sourceBlockIds) ? card.sourceBlockIds : [];
|
const rawIds: string[] = Array.isArray(card.sourceBlockIds) ? card.sourceBlockIds : [];
|
||||||
const validIds = rawIds.filter(id => existingBlockIds.has(id));
|
const validIds = rawIds.filter(id => existingBlockIds.has(id));
|
||||||
@ -528,14 +542,8 @@ export class RuntimeInternalService {
|
|||||||
this.logger.warn(`convertFlashcardCandidates: filtered ${rawIds.length - validIds.length} invalid sourceBlockIds`);
|
this.logger.warn(`convertFlashcardCandidates: filtered ${rawIds.length - validIds.length} invalid sourceBlockIds`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Dedup: skip if same front already exists for this user
|
|
||||||
const front = card.front ?? card.question;
|
const front = card.front ?? card.question;
|
||||||
const dupCheck = await this.prisma.flashcard.findFirst({
|
if (existingFronts.has(front)) {
|
||||||
where: { userId: job.userId, front, deletedAt: null },
|
|
||||||
select: { id: true },
|
|
||||||
});
|
|
||||||
if (dupCheck) {
|
|
||||||
this.logger.warn(`convertFlashcardCandidates: skipping duplicate front="${front.substring(0, 50)}"`);
|
this.logger.warn(`convertFlashcardCandidates: skipping duplicate front="${front.substring(0, 50)}"`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user