use MangaDex to better find links between AniList and MangaUpdates

frontend/src/components/navbar/MangaUpdatesUpdater.vue

@@ -1,5 +1,5 @@
 <script lang="ts">
-import {Progress} from '@/data/service/MangaUpdatesDataService';
+import {Progress} from '@/data/service/Progress';
 import {Options, Vue} from 'vue-class-component';
 import {ServiceStore} from '@/stores/ServiceStore';
 import {BSpinner} from 'bootstrap-vue-next';
@@ -13,7 +13,8 @@ import {toast} from 'vue3-toastify';
 })
 export default class MangaUpdatesUpdater extends Vue {
   //vars
-  progress: Progress = new Progress(this.onProgress.bind(this), this.onFinished.bind(this));
+  progressMangaDex: Progress = new Progress(this.onProgress.bind(this), () => {});
+  progressMangaUpdates: Progress = new Progress(this.onProgress.bind(this), this.onFinished.bind(this));
   progressType: string | null = null;
   progressValue: number | null = null;
   progressMax: number | null = null;
@@ -35,9 +36,10 @@ export default class MangaUpdatesUpdater extends Vue {
 
   //event handler
   async onUpdateMangaUpdatesDb(): Promise<void> {
-    this.progressType = 'starting';
+    this.progressType = 'general.starting';
     await this.tryAcquireWakeLock();
-    this.serviceStore.mangaUpdatesDataService.updateDb(this.progress);
+    await this.serviceStore.mangaDexDataService.updateDb(this.progressMangaDex);
+    await this.serviceStore.mangaUpdatesDataService.updateDb(this.progressMangaUpdates);
   };
 
   onProgress(type: string, progress: number, max: number): void {
@@ -50,7 +52,7 @@ export default class MangaUpdatesUpdater extends Vue {
     this.wakeLock?.release();
     this.wakeLock = null;
 
-    this.progressType = 'finished';
+    this.progressType = 'general.finished';
     this.progressValue = null;
     this.progressMax = null;
 
@@ -89,10 +91,10 @@ export default class MangaUpdatesUpdater extends Vue {
       <span class="text-nowrap">
         <BSpinner small class="me-2"/>
         <template v-if="progressValue === null || progressMax === null">
-          {{ $t('fetch.mangaUpdates.' + progressType) }}
+          {{ $t('fetch.' + progressType) }}
         </template>
         <template v-else>
-          {{ $t('fetch.mangaUpdates.' + progressType) + ': ' + (progressValue ?? 0) + '/' + (progressMax ?? 1) }}
+          {{ $t('fetch.' + progressType) + ': ' + (progressValue ?? 0) + '/' + (progressMax ?? 1) }}
         </template>
       </span>
     </template>

frontend/src/data/api/MangaDexApi.ts

@@ -0,0 +1,9 @@
+import {handleJsonResponse} from '@/data/api/ApiUtils';
+import type {MangaDexSearchResult} from '@/data/models/mangadex/MangaDexSearchResult';
+
+export default class MangaDexApi {
+  search(name: string): Promise<MangaDexSearchResult> {
+    const res = fetch('/mangadex/manga?title=' + encodeURIComponent(name));
+    return handleJsonResponse(res);
+  }
+}

frontend/src/data/api/MangaUpdatesApi.ts

@@ -3,6 +3,9 @@ import {handleJsonResponse} from '@/data/api/ApiUtils';
 import type {MangaUpdatesSearchResult} from '@/data/models/mangaupdates/MangaUpdatesSearchResult';
 import type {MangaUpdatesSeries} from '@/data/models/mangaupdates/MangaUpdatesSeries';
 import type {MangaUpdatesSeriesGroups} from '@/data/models/mangaupdates/MangaUpdatesSeriesGroups';
+import type {
+  MangaUpdatesWebsiteIdSeriesIdRelation
+} from '@/data/models/mangaupdates/MangaUpdatesWebsiteIdSeriesIdRelation';
 
 export default class MangaUpdatesApi {
   search(name: string): Promise<MangaUpdatesSearchResult> {
@@ -27,4 +30,9 @@ export default class MangaUpdatesApi {
     const res = fetch(`/mangaupdates/v1/series/${id}/groups`);
     return handleJsonResponse(res);
   }
+
+  seriesIdByWebsiteId(websiteId: number): Promise<MangaUpdatesWebsiteIdSeriesIdRelation> {
+    const res = fetch('/mangaupdates/series_id_from_website_id/' + websiteId);
+    return handleJsonResponse(res);
+  }
 }

frontend/src/data/models/mangadex/MangaDexMedia.ts

@@ -0,0 +1,66 @@
+export type MangaDexMedia = {
+  id: string, //uuid
+  type: 'manga' | string,
+  attributes: {
+    title: {
+      //2 letter language code: name
+      [key: string]: string,
+    },
+    altTitles: {
+      //2 letter language code: name
+      [key: string]: string
+    }[],
+    description: {
+      //2 letter language code: description
+      [key: string]: string
+    },
+    isLocked: boolean,
+    links?: {
+      al?: string, //anilist id
+      ap?: string, //human-readable title,
+      bw?: string, //'series/<id>',
+      kt?: string, //kt id
+      mu?: string, //mangaupdates id
+      nu?: string, //human-readable title,
+      amz?: string, //url
+      ebj?: string, //url
+      mal?: string, //myanimelist id
+      raw?: string, //url
+      engtl?: string, //url
+      [key: string]: string | undefined,
+    },
+    originalLanguage: string,//2 letter language code
+    lastVolume: string,
+    lastChapter: string,
+    publicationDemographic: 'shounen' | string,
+    status: 'ongoing' | string,
+    year: number,
+    contentRating: 'suggestive' | string,
+    tags: {
+      id: string, //uuid
+      type: 'tag',
+      attributes: {
+        name: {
+          //2 letter language code: name
+          [key: string]: string
+        },
+        description: {},
+        group: 'theme' | string,
+        version: number
+      },
+      relationships: []
+    }[],
+    state: 'published' | string,
+    chapterNumbersResetOnNewVolume: boolean,
+    createdAt: string, //iso date string
+    updatedAt: string, //iso date string
+    version: number,
+    availableTranslatedLanguages: string[], //2 letter language code
+    latestUploadedChapter: string, //uuid
+  },
+  relationships: {
+    id: string, //uuid
+    type: 'author' | 'artist' | 'cover_art' | 'adapted_from' | 'side_story' | string,
+    related?: 'main_story' | string,
+  }[]
+}

frontend/src/data/models/mangadex/MangaDexSearchResult.ts

@@ -0,0 +1,10 @@
+import type {MangaDexMedia} from '@/data/models/mangadex/MangaDexMedia';
+
+export type MangaDexSearchResult = {
+  result: 'ok' | string,
+  response: 'collection' | string,
+  data: MangaDexMedia[],
+  limit: number,
+  offset: number,
+  total: number
+}

frontend/src/data/models/mangaupdates/MangaUpdatesWebsiteIdSeriesIdRelation.ts

@@ -0,0 +1,4 @@
+export type MangaUpdatesWebsiteIdSeriesIdRelation = {
+  website_id: string;
+  series_id: number;
+}

frontend/src/data/service/MangaDexDataService.ts

@@ -0,0 +1,84 @@
+import {MangaStore} from '@/stores/MangaStore';
+import {DbStore} from '@/stores/DbStore';
+import {ApiError} from '@/data/api/ApiUtils';
+import MangaDexApi from '@/data/api/MangaDexApi';
+import {Progress} from '@/data/service/Progress';
+import MangaUpdatesApi from '@/data/api/MangaUpdatesApi';
+import type {MangaUpdatesRelation} from '@/data/models/mangaupdates/MangaUpdatesRelation';
+
+export default class MangaDexDataService {
+  private readonly mangaDexApi = new MangaDexApi();
+  private readonly mangaUpdatesApi = new MangaUpdatesApi();
+
+  updateDb(progress: Progress): Promise<void> {
+    const mangaStore = new MangaStore();
+    const dbStore = new DbStore();
+    return this.findMissingRelations(mangaStore, dbStore, progress)
+      .catch(err => console.error(err))
+      .then(_ => progress.onFinished());
+  }
+
+  private async findMissingRelations(mangaStore: MangaStore, dbStore: DbStore, progress: Progress): Promise<void> {
+    const allowedTypes = new Set(['manga'].map(e => e.toLowerCase()));
+
+    const media = await dbStore.aniListMangaRepository.getMedia();
+    const relations = await dbStore.mangaUpdatesRepository.getRelations();
+    const presentRelationIds = new Set(relations.map(e => e.aniListMediaId));
+
+    let i = 0;
+    for (const m of media) {
+      try {
+        if (presentRelationIds.has(m.id)) {
+          continue;
+        }
+
+        let matching: MangaUpdatesRelation[] = [];
+        for (let title of [m.title.native, m.title.romaji, m.title.english]) {
+          if (!title?.trim().length) {
+            continue;
+          }
+          let results;
+          try {
+            results = await this.mangaDexApi.search(title);
+          } catch (err) {
+            if (err instanceof ApiError && [400, 500].includes(err.statusCode)) {
+              console.debug(err);
+            } else {
+              console.error(err);
+            }
+            continue;
+          }
+          matching = results.data
+            .filter(e => allowedTypes.has(e.type))
+            .filter(e => e.attributes.links?.al?.match(/[0-9]+/) && e.attributes.links.mu)
+            .filter(e => parseInt(e.attributes.links!.al!) === m.id)
+            .map(e => ({
+              aniListMediaId: parseInt(e.attributes.links!.al!),
+              mangaUpdatesSeriesId: e.attributes.links!.mu! as any as number, //TODO cast is hack, make pretty
+            }));
+
+          for (let j = 0; j < matching.length; j++) {
+            try {
+              const websiteIdSeriesIdRelation = await this.mangaUpdatesApi.seriesIdByWebsiteId(matching[j].mangaUpdatesSeriesId);
+              matching[j].mangaUpdatesSeriesId = websiteIdSeriesIdRelation.series_id;
+            } catch(_) {
+              matching.splice(j--, 1);
+            }
+          }
+
+          if (matching.length > 0) {
+            break;
+          }
+        }
+
+        if (matching.length === 0) {
+          continue;
+        }
+
+        await mangaStore.addMangaUpdatesRelations([matching[0]]);
+      } finally {
+        await progress.onProgress('mangaDex.relations', ++i, media.length);
+      }
+    }
+  }
+}

frontend/src/data/service/MangaUpdatesDataService.ts

@@ -7,14 +7,15 @@ import stringSimilarity from 'string-similarity-js';
 import {ApiError} from '@/data/api/ApiUtils';
 import groupBy from '@/util';
 import {decode} from 'html-entities';
+import {Progress} from '@/data/service/Progress';
 
 export default class MangaUpdatesDataService {
   private readonly mangaUpdatesApi = new MangaUpdatesApi();
 
-  updateDb(progress: Progress): Progress {
+  updateDb(progress: Progress): Promise<void> {
     const mangaStore = new MangaStore();
     const dbStore = new DbStore();
-    this.findMissingRelations(mangaStore, dbStore, progress)
+    return this.findMissingRelations(mangaStore, dbStore, progress)
       .catch(err => {
         console.error(err);
       })
@@ -27,7 +28,6 @@ export default class MangaUpdatesDataService {
         console.error(err);
       })
       .then(_ => progress.onFinished());
-    return progress;
   }
 
   private async findMissingRelations(mangaStore: MangaStore, dbStore: DbStore, progress: Progress): Promise<void> {
@@ -79,7 +79,7 @@ export default class MangaUpdatesDataService {
 
         await mangaStore.addMangaUpdatesRelations([{aniListMediaId: m.id, mangaUpdatesSeriesId: matching[0].record.series_id}]);
       } finally {
-        await progress.onProgress('relations', ++i, media.length);
+        await progress.onProgress('mangaUpdates.relations', ++i, media.length);
       }
     }
   }
@@ -108,7 +108,7 @@ export default class MangaUpdatesDataService {
         }
         await mangaStore.updateMangaUpdatesSeries([series]);
       } finally {
-        await progress.onProgress('series', ++i, relations.length);
+        await progress.onProgress('mangaUpdates.series', ++i, relations.length);
       }
     }
   }
@@ -148,30 +148,10 @@ export default class MangaUpdatesDataService {
           .map(chaptersOfGroup => chaptersOfGroup.reduce((l, r) => l.chapter > r.chapter ? l : r, chaptersOfGroup[0]));
         cachedChapterUpdates.push(...filtered)
       } finally {
-        await progress.onProgress('chapters', ++i, series.length);
+        await progress.onProgress('mangaUpdates.chapters', ++i, series.length);
       }
     }
     await mangaStore.updateMangaUpdatesChapters(cachedChapterUpdates);
   }
 }
 
-export type ProgressCallBackFn = (type: string, progress: number, max: number) => (Promise<void> | void);
-export type FinishCallBackFn = () => (Promise<void> | void);
-
-export class Progress {
-  readonly progress: ProgressCallBackFn;
-  readonly finish: FinishCallBackFn;
-
-  constructor(onProgress: ProgressCallBackFn, onFinish: FinishCallBackFn) {
-    this.progress = onProgress;
-    this.finish = onFinish;
-  }
-
-  async onProgress(type: string, progress: number, max: number): Promise<void> {
-    return (this.progress ? this.progress(type, progress, max) : Promise.resolve());
-  }
-
-  async onFinished(): Promise<void> {
-    return (this.finish ? this.finish() : Promise.resolve());
-  }
-}

frontend/src/data/service/Progress.ts

@@ -0,0 +1,20 @@
+export type ProgressCallBackFn = (type: string, progress: number, max: number) => (Promise<void> | void);
+export type FinishCallBackFn = () => (Promise<void> | void);
+
+export class Progress {
+  readonly progress: ProgressCallBackFn;
+  readonly finish: FinishCallBackFn;
+
+  constructor(onProgress: ProgressCallBackFn, onFinish: FinishCallBackFn) {
+    this.progress = onProgress;
+    this.finish = onFinish;
+  }
+
+  async onProgress(type: string, progress: number, max: number): Promise<void> {
+    return (this.progress ? this.progress(type, progress, max) : Promise.resolve());
+  }
+
+  async onFinished(): Promise<void> {
+    return (this.finish ? this.finish() : Promise.resolve());
+  }
+}

frontend/src/locale/de.ts

@@ -6,12 +6,17 @@ export const messagesDe = {
   },
   design: 'Design',
   fetch: {
+    general: {
+      finished: 'Fertig',
+      starting: 'Starte',
+    },
+    mangaDex: {
+      relations: 'Relationen',
+    },
     mangaUpdates: {
       chapters: 'Kapitel',
-      finished: 'Fertig',
       relations: 'Relationen',
       series: 'Kapitel',
-      starting: 'Starte',
     },
   },
   locale: 'Sprache',

frontend/src/locale/en.ts

@@ -4,12 +4,17 @@ export const messagesEn = {
   },
   design: 'Design',
   fetch: {
+    general: {
+      finished: 'Finished',
+      starting: 'Starting',
+    },
+    mangaDex: {
+      relations: 'Relations',
+    },
     mangaUpdates: {
       chapters: 'Chapters',
-      finished: 'Finished',
       relations: 'Relations',
       series: 'Series',
-      starting: 'Starting',
     },
   },
   locale: 'Language',

frontend/src/stores/ServiceStore.ts

@@ -1,5 +1,6 @@
 import {Pinia, Store} from 'pinia-class-component';
 import AniListDataService from '@/data/service/AniListDataService';
+import MangaDexDataService from '@/data/service/MangaDexDataService';
 import MangaUpdatesDataService from '@/data/service/MangaUpdatesDataService';
 
 @Store({
@@ -9,6 +10,7 @@ import MangaUpdatesDataService from '@/data/service/MangaUpdatesDataService';
 export class ServiceStore extends Pinia {
   //data
   private readonly intAniListDataService = new AniListDataService();
+  private readonly intMangaDexDataService = new MangaDexDataService();
   private readonly intMangaUpdatesDataService = new MangaUpdatesDataService();
 
   //getter
@@ -16,6 +18,10 @@ export class ServiceStore extends Pinia {
     return this.intAniListDataService;
   }
 
+  get mangaDexDataService(): MangaDexDataService {
+    return this.intMangaDexDataService;
+  }
+
   get mangaUpdatesDataService(): MangaUpdatesDataService {
     return this.intMangaUpdatesDataService;
   }