added backend to cache manga updates requests

backend/src/cache/MangaUpdatesCache.ts

@@ -0,0 +1,98 @@
+import * as fs from 'fs';
+
+type CacheEntry = { data: string, lastUpdateMs: number }
+
+export class MangaUpdatesCache {
+  private readonly FILE_CACHE_DIR = 'cache';
+  private readonly FILE_SEARCH_BY_TITLE = this.FILE_CACHE_DIR + '/searchByTitle.json';
+  private readonly FILE_SERIES_BY_ID = this.FILE_CACHE_DIR + '/seriesById.json';
+  private readonly FILE_SERIES_GROUPS_BY_ID = this.FILE_CACHE_DIR + '/seriesGroupsById.json';
+
+  private readonly MAX_CACHE_AGE_SEARCH_BY_TITLE = 7 * 24 * 60 * 60 * 1000;
+  private readonly MAX_CACHE_AGE_SERIES_BY_ID = 30 * 24 * 60 * 60 * 1000;
+  private readonly MAX_CACHE_AGE_SERIES_GROUPS_BY_ID = 1 * 24 * 60 * 60 * 1000;
+
+  private _searchByTitle = new Map<string, CacheEntry>();
+  private _seriesById = new Map<string, CacheEntry>();
+  private _seriesGroupsById = new Map<string, CacheEntry>();
+
+  constructor() {
+    this.load();
+  }
+
+  private load(): void {
+    try {
+      fs.mkdirSync(this.FILE_CACHE_DIR);
+    } catch (_) {
+    }
+    this._searchByTitle = this.loadJson(this.FILE_SEARCH_BY_TITLE);
+    this._seriesById = this.loadJson(this.FILE_SERIES_BY_ID);
+    this._seriesGroupsById = this.loadJson(this.FILE_SERIES_GROUPS_BY_ID);
+  }
+
+  private loadJson(file: string): Map<string, CacheEntry> {
+    try {
+      const data = fs.readFileSync(file, {encoding: 'utf8', flag: 'r'});
+      const json = JSON.parse(data);
+      return new Map<string, CacheEntry>(Object.entries(json));
+    } catch (err) {
+      console.error('Failed to load cache from ' + file + ':' + (err as Error).message);
+      return new Map();
+    }
+  }
+
+  private saveJson(file: string, map: Map<string, CacheEntry>): void {
+    try {
+      const data = JSON.stringify(Object.fromEntries(map.entries()));
+      fs.writeFileSync(file, data, {encoding: 'utf8', flag: 'w'});
+    } catch (err) {
+      console.error(err);
+    }
+  }
+
+  getSearchByTitle(title: string): string | undefined {
+    const entry = this._searchByTitle.get(title);
+    return !entry || entry.lastUpdateMs + this.MAX_CACHE_AGE_SEARCH_BY_TITLE < Date.now() ? undefined : entry.data;
+  }
+
+  getSeriesById(id: string): string | undefined {
+    const entry = this._seriesById.get(id);
+    return !entry || entry.lastUpdateMs + this.MAX_CACHE_AGE_SERIES_BY_ID < Date.now() ? undefined : entry.data;
+  }
+
+  getSeriesGroupsById(id: string): string | undefined {
+    const entry = this._seriesGroupsById.get(id);
+    return !entry || entry.lastUpdateMs + this.MAX_CACHE_AGE_SERIES_GROUPS_BY_ID < Date.now() ? undefined : entry.data;
+  }
+
+  putSearchByTitle(title: string, value: string): void {
+    this._searchByTitle.set(title, {data: value, lastUpdateMs: Date.now()});
+    this.saveJson(this.FILE_SEARCH_BY_TITLE, this._searchByTitle);
+  }
+
+  putSeriesById(id: string, value: string): void {
+    this._seriesById.set(id, {data: value, lastUpdateMs: Date.now()});
+    this.saveJson(this.FILE_SERIES_BY_ID, this._seriesById);
+  }
+
+  putSeriesGroupsById(id: string, value: string): void {
+    this._seriesGroupsById.set(id, {data: value, lastUpdateMs: Date.now()});
+    this.saveJson(this.FILE_SERIES_GROUPS_BY_ID, this._seriesGroupsById);
+  }
+
+  getOutOfDateSearch(): string[] {
+    return Array.from(this._searchByTitle.entries())
+      .filter(([title, entry]) => entry.lastUpdateMs + this.MAX_CACHE_AGE_SEARCH_BY_TITLE < Date.now())
+      .map(([title, entry]) => title);
+  }
+  getOutOfDateSeries(): string[] {
+    return Array.from(this._seriesById.entries())
+      .filter(([title, entry]) => entry.lastUpdateMs + this.MAX_CACHE_AGE_SERIES_BY_ID < Date.now())
+      .map(([title, entry]) => title);
+  }
+  getOutOfDateSeriesGroups(): string[] {
+    return Array.from(this._seriesGroupsById.entries())
+      .filter(([title, entry]) => entry.lastUpdateMs + this.MAX_CACHE_AGE_SERIES_GROUPS_BY_ID < Date.now())
+      .map(([title, entry]) => title);
+  }
+}

backend/src/controller/MangaUpdatesController.ts

@@ -0,0 +1,106 @@
+import {NextFunction, Request, Response} from 'express';
+import {MangaUpdatesCache} from '../cache/MangaUpdatesCache.js';
+
+export class MangaUpdatesController {
+  private cache: MangaUpdatesCache;
+
+  constructor(cache: MangaUpdatesCache) {
+    this.cache = cache;
+  }
+
+  async search(req: Request, res: Response, next: NextFunction): Promise<void> {
+    const bjson = req.body;
+    if (bjson['stype'] !== 'title' || bjson['type'] !== 'Manga' || !bjson['search']?.trim().length) {
+      res.status(400).send('Only {stype: "title", type: "Manga", search: "some title"} allowed!');
+      next('router');
+      return;
+    }
+
+    const fromCache = this.cache.getSearchByTitle(bjson['search']);
+    if (fromCache) {
+      res.status(200).setHeader('Content-Type', 'application/json').send(fromCache);
+      next('router');
+      return;
+    }
+
+    //throttle
+    await new Promise((r) => setTimeout(r, 1000));
+
+    //fetch from manga updates
+    const fromApi = await fetch('https://api.mangaupdates.com/v1/series/search', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(bjson),
+    });
+
+    if (fromApi.status !== 200) {
+      res.status(fromApi.status).send(fromApi.body);
+      next('router');
+      return;
+    }
+
+    const fromApiJson = await fromApi.text();
+    this.cache.putSearchByTitle(bjson['search'], fromApiJson);
+
+    res.status(200).setHeader('Content-Type', 'application/json').send(fromApiJson);
+    next('router');
+  }
+
+  async getById(req: Request, res: Response, next: NextFunction): Promise<void> {
+    const id = req.params.id!.toLowerCase();
+
+    const fromCache = this.cache.getSeriesById(id);
+    if (fromCache) {
+      res.status(200).setHeader('Content-Type', 'application/json').send(fromCache);
+      next();
+      return;
+    }
+
+    //throttle
+    await new Promise((r) => setTimeout(r, 1000));
+
+    //fetch from manga updates
+    const fromApi = await fetch('https://api.mangaupdates.com/v1/series/' + id);
+    if (fromApi.status !== 200) {
+      res.status(fromApi.status).send(fromApi.body);
+      next();
+      return;
+    }
+
+    const fromApiJson = await fromApi.text();
+    this.cache.putSeriesById(id, fromApiJson);
+
+    res.status(200).setHeader('Content-Type', 'application/json').send(fromApiJson);
+    next();
+  }
+
+  async getGroupById(req: Request, res: Response, next: NextFunction): Promise<void> {
+    const id = req.params.id!.toLowerCase();
+
+    const fromCache = this.cache.getSeriesGroupsById(id);
+    if (fromCache) {
+      res.status(200).setHeader('Content-Type', 'application/json').send(fromCache);
+      next();
+      return;
+    }
+
+    //throttle
+    await new Promise((r) => setTimeout(r, 1000));
+
+    //fetch from manga updates
+    const fromApi = await fetch('https://api.mangaupdates.com/v1/series/' + id + '/groups');
+    if (fromApi.status !== 200) {
+      res.status(fromApi.status).send(fromApi.body);
+      next();
+      return;
+    }
+
+    const fromApiJson = await fromApi.text();
+    this.cache.putSeriesGroupsById(id, fromApiJson);
+
+    res.status(200).setHeader('Content-Type', 'application/json').send(fromApiJson);
+    next();
+  }
+}

backend/src/main.ts

@@ -0,0 +1,29 @@
+import compression from 'compression';
+import express from 'express';
+import aniListRouter from './router/AniListRouter.js';
+import mangaUpdatesRouter from './router/MangaUpdatesRouter.js';
+import Scheduler from './schedule/Scheduler.js';
+import {MangaUpdatesCache} from './cache/MangaUpdatesCache.js';
+import * as fs from 'fs';
+
+const config = JSON.parse(fs.readFileSync('config.json').toString())
+
+const app = express();
+const mangaUpdatesCache = new MangaUpdatesCache();
+const scheduler = new Scheduler(mangaUpdatesCache);
+
+scheduler.registerJobs();
+
+//middlewares
+app.use(compression());
+app.use(express.json());
+
+//router
+app.use('/anilist', aniListRouter());
+app.use('/mangaupdates', mangaUpdatesRouter(mangaUpdatesCache));
+app.use(express.static('_client')) //for production
+
+//start
+app.listen(config.port, config.host, () => {
+  console.log('Started server on http://' + config.host + ':' + config.port + '/');
+});

backend/src/router/AniListRouter.ts

@@ -0,0 +1,24 @@
+import {NextFunction, Request, Response, Router} from 'express';
+
+export default function aniListRouter(): Router {
+  const router = Router();
+  router.post('/graphql', async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const fromApi = await fetch('https://graphql.anilist.co/graphql', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(req.body),
+      });
+
+      const fromApiText = await fromApi.text()
+      res.status(fromApi.status).send(fromApiText);
+    } catch (e) {
+      console.error(e);
+    }
+    next();
+  });
+
+  return router;
+}

backend/src/router/MangaUpdatesRouter.ts

@@ -0,0 +1,12 @@
+import {Router} from 'express';
+import {MangaUpdatesController} from '../controller/MangaUpdatesController.js';
+import {MangaUpdatesCache} from '../cache/MangaUpdatesCache.js';
+
+export default function mangaUpdatesRouter(cache: MangaUpdatesCache): Router {
+  const controller = new MangaUpdatesController(cache);
+  const router = Router();
+  router.post('/v1/series/search', controller.search.bind(controller));
+  router.get('/v1/series/:id', controller.getById.bind(controller));
+  router.get('/v1/series/:id/groups', controller.getGroupById.bind(controller));
+  return router;
+}

backend/src/schedule/IJob.ts

@@ -0,0 +1,4 @@
+export default interface IJob<T> {
+  get schedule(): Date | string;
+  execute(): Promise<T>;
+}

backend/src/schedule/MangaUpdateCacheRenewJob.ts

@@ -0,0 +1,28 @@
+import IJob from './IJob';
+import MangaUpdateCacheRenewService from '../service/MangaUpdateCacheRenewService.js';
+import {MangaUpdatesCache} from '../cache/MangaUpdatesCache.js';
+
+export default class MangaUpdateCacheRenewJob implements IJob<void> {
+  private readonly service: MangaUpdateCacheRenewService;
+  private lock: boolean = false;
+
+  constructor(cache: MangaUpdatesCache) {
+    this.service = new MangaUpdateCacheRenewService(cache);
+  }
+
+  get schedule(): Date | string {
+    return '0 0 * * * *'; //every hour
+  }
+
+  async execute(): Promise<void> {
+    if (this.lock) {
+      return;
+    }
+    this.lock = true;
+    try {
+      await this.service.renew();
+    } finally {
+      this.lock = false;
+    }
+  }
+}

backend/src/schedule/Scheduler.ts

@@ -0,0 +1,22 @@
+import {MangaUpdatesCache} from '../cache/MangaUpdatesCache.js';
+import IJob from './IJob.js';
+import MangaUpdateCacheRenewJob from './MangaUpdateCacheRenewJob.js';
+import {gracefulShutdown, scheduleJob} from 'node-schedule';
+
+export default class Scheduler {
+  private readonly jobs: IJob<any>[] = [];
+
+  constructor(cache: MangaUpdatesCache) {
+    this.jobs.push(
+      new MangaUpdateCacheRenewJob(cache),
+    );
+  }
+
+  registerJobs(): void {
+    this.jobs.forEach(e => scheduleJob(e.schedule, e.execute.bind(e)));
+  }
+
+  cancelJobs(): Promise<void> {
+    return gracefulShutdown();
+  }
+}

backend/src/service/MangaUpdateCacheRenewService.ts

@@ -0,0 +1,86 @@
+import {MangaUpdatesCache} from '../cache/MangaUpdatesCache';
+
+export default class MangaUpdateCacheRenewService {
+  private static readonly delay = 3000;
+
+  private readonly cache: MangaUpdatesCache;
+
+  constructor(cache: MangaUpdatesCache) {
+    this.cache = cache;
+  }
+
+  async renew(): Promise<void> {
+    console.log('Renewing cache ...');
+    await this.renewRelations();
+    await this.renewSeries();
+    await this.renewUpdates();
+    console.log('Renewing cache done');
+  }
+
+  async renewRelations(): Promise<void> {
+    const titles = this.cache.getOutOfDateSearch();
+    console.log(titles.length + ' out-of-date relations');
+
+    for (let title of titles) {
+      await new Promise((r) => setTimeout(r, MangaUpdateCacheRenewService.delay));
+      try {
+        const fromApi = await fetch('https://api.mangaupdates.com/v1/series/search', {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({stype: 'title', type: 'Manga', search: title}),
+        });
+
+        if (fromApi.status !== 200) {
+          continue;
+        }
+
+        const fromApiJson = await fromApi.text();
+        this.cache.putSearchByTitle(title, fromApiJson);
+      } catch (e) {
+        console.error(e);
+      }
+    }
+  }
+
+  async renewSeries(): Promise<void> {
+    const ids = this.cache.getOutOfDateSeries();
+    console.log(ids.length + ' out-of-date series');
+
+    for (let id of ids) {
+      await new Promise((r) => setTimeout(r, MangaUpdateCacheRenewService.delay));
+      try {
+        const fromApi = await fetch('https://api.mangaupdates.com/v1/series/' + id);
+        if (fromApi.status !== 200) {
+          continue;
+        }
+
+        const fromApiJson = await fromApi.text();
+        this.cache.putSeriesById(id, fromApiJson);
+      } catch (e) {
+        console.error(e);
+      }
+    }
+  }
+
+  async renewUpdates(): Promise<void> {
+    const ids = this.cache.getOutOfDateSeriesGroups();
+    console.log(ids.length + ' out-of-date series updates');
+
+    for (let id of ids) {
+      await new Promise((r) => setTimeout(r, MangaUpdateCacheRenewService.delay));
+      try {
+        const fromApi = await fetch('https://api.mangaupdates.com/v1/series/' + id + '/groups');
+        if (fromApi.status !== 200) {
+          continue;
+        }
+
+        const fromApiJson = await fromApi.text();
+        this.cache.putSeriesGroupsById(id, fromApiJson);
+      } catch (e) {
+        console.error(e);
+      }
+    }
+  }
+}