import fetch from 'node-fetch' import { ApiKeyModel } from '../models/apikey.model.ts' import { AssignmentModel } from '../models/assignments.model.ts' import { KanjiModel } from '../models/kanji.model.ts' import { VocabularyModel } from '../models/vocabulary.model.ts' import type { KanjiItem, VocabularyItem } from '../types/wanikani.ts' const WANIKANI_API_BASE = 'https://api.wanikani.com/v2' export interface WaniKaniSubject { id: number object: string data: { characters: string meanings: { meaning: string; primary: boolean; accepted_answers: boolean }[] readings: { type: string; primary: boolean; accepted_answer: boolean; reading: string }[] auxiliary_meanings: { meaning: string; type: string }[] pronunciation_audios?: { url: string content_type: string metadata: { gender: string; pronunciation: string }[] }[] level: number slug: string unlocked_at?: string } } const fetchAllPages = async ( url: string, headers: Record, ): Promise => { let results: WaniKaniSubject[] = [] let nextUrl: string | null = url while (nextUrl) { const res = await fetch(nextUrl, { headers }) if (!res.ok) throw new Error(`Failed fetching ${url}: ${res.statusText}`) const json = (await res.json()) as { data: WaniKaniSubject[]; pages: { next_url: string | null } } results = results.concat(json.data) nextUrl = json.pages.next_url } return results } const fetchSubjects = async ( ids: number[], headers: Record, ): Promise => { const chunkSize = 1000 let results: WaniKaniSubject[] = [] for (let i = 0; i < ids.length; i += chunkSize) { const chunk = ids.slice(i, i + chunkSize) const res = await fetch(`${WANIKANI_API_BASE}/subjects?ids=${chunk.join(',')}`, { headers }) if (!res.ok) throw new Error(`Failed fetching subjects: ${res.statusText}`) const json = (await res.json()) as { data: WaniKaniSubject[] } results = results.concat(json.data) } return results } const mapKanji = (item: WaniKaniSubject): KanjiItem => ({ characters: item.data.characters, meanings: item.data.meanings, readings: item.data.readings, auxiliary_meanings: item.data.auxiliary_meanings, level: item.data.level, slug: item.data.slug, srs_score: 0, }) const mapVocab = (item: WaniKaniSubject): VocabularyItem => ({ characters: item.data.characters, meanings: item.data.meanings, readings: item.data.readings ?? [], auxiliary_meanings: item.data.auxiliary_meanings, pronunciation_audios: item.data.pronunciation_audios ?? [], level: item.data.level, slug: item.data.slug, srs_score: 0, }) export const syncWanikaniData = async (apiKey: string): Promise => { const headers = { Authorization: `Bearer ${apiKey}` } try { await ApiKeyModel.updateOne( {}, { $set: { value: apiKey, lastUsed: new Date() } }, { upsert: true }, ) const assignments = await fetchAllPages(`${WANIKANI_API_BASE}/assignments`, headers) const unlockedKanjiSubjectIds: number[] = [] const unlockedVocabSubjectIds: number[] = [] for (const a of assignments) { const assignmentData = a.data as any if (!assignmentData.unlocked_at) continue if (assignmentData.subject_type === 'kanji') { unlockedKanjiSubjectIds.push(assignmentData.subject_id) } else if (['vocabulary', 'kana_vocabulary'].includes(assignmentData.subject_type)) { unlockedVocabSubjectIds.push(assignmentData.subject_id) } } await AssignmentModel.updateOne( { subject_type: 'kanji' }, { $set: { subject_ids: unlockedKanjiSubjectIds } }, { upsert: true }, ) await AssignmentModel.updateOne( { subject_type: 'vocabulary' }, { $set: { subject_ids: unlockedVocabSubjectIds } }, { upsert: true }, ) const existingKanjiSlugs = new Set((await KanjiModel.find({}, { slug: 1 })).map(k => k.slug)) const kanjiSubjects = await fetchSubjects(unlockedKanjiSubjectIds, headers) const newKanji = kanjiSubjects.filter(s => !existingKanjiSlugs.has(s.data.slug)) if (newKanji.length > 0) await KanjiModel.insertMany(newKanji.map(mapKanji)) const existingVocabSlugs = new Set((await VocabularyModel.find({}, { slug: 1 })).map(v => v.slug)) const vocabSubjects = await fetchSubjects(unlockedVocabSubjectIds, headers) const newVocab = vocabSubjects.filter(s => !existingVocabSlugs.has(s.data.slug)) if (newVocab.length > 0) await VocabularyModel.insertMany(newVocab.map(mapVocab)) console.log('✅ Sync complete') } catch (err) { console.error('❌ Error syncing WaniKani data:', err) throw err } }