wea_ondara/MangaStatus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use MangaDex to better find links between AniList and MangaUpdates

Browse Source
This commit is contained in:
wea_ondara
2023-11-25 16:58:48 +01:00
parent 53f059b7de
commit d8bb97805c
25 changed files with 506 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
backend/src/cache/CacheUtil.ts vendored Normal file
View File

@@ -0,0 +1,23 @@
import * as fs from 'fs';
export type CacheEntry = { data: string, lastUpdateMs: number }
export function loadJson(file: string): Map<string, CacheEntry> {
try {
const data = fs.readFileSync(file, {encoding: 'utf8', flag: 'r'});
const json = JSON.parse(data);
return new Map<string, CacheEntry>(Object.entries(json));
} catch (err) {
console.error('Failed to load cache from ' + file + ':' + (err as Error).message);
return new Map();
}
}
export function saveJson(file: string, map: Map<string, CacheEntry>): void {
try {
const data = JSON.stringify(Object.fromEntries(map.entries()));
fs.writeFileSync(file, data, {encoding: 'utf8', flag: 'w'});
} catch (err) {
console.error(err);
}
}

39
backend/src/cache/MangaDexCache.ts vendored Normal file
View File

@@ -0,0 +1,39 @@
import * as fs from 'fs';
import {CacheEntry, loadJson, saveJson} from './CacheUtil';
export class MangaDexCache {
private readonly FILE_CACHE_DIR = 'cache/mangadex';
private readonly FILE_SEARCH_BY_TITLE = this.FILE_CACHE_DIR + '/searchByTitle.json';
private readonly MAX_CACHE_AGE_SEARCH_BY_TITLE = 7 * 24 * 60 * 60 * 1000;
private _searchByTitle = new Map<string, CacheEntry>();
constructor() {
this.load();
}
private load(): void {
try {
fs.mkdirSync(this.FILE_CACHE_DIR, {recursive: true});
} catch (_) {
}
this._searchByTitle = loadJson(this.FILE_SEARCH_BY_TITLE);
}
getSearchByTitle(title: string): string | undefined {
const entry = this._searchByTitle.get(title);
return !entry || entry.lastUpdateMs + this.MAX_CACHE_AGE_SEARCH_BY_TITLE < Date.now() ? undefined : entry.data;
}
putSearchByTitle(title: string, value: string): void {
this._searchByTitle.set(title, {data: value, lastUpdateMs: Date.now()});
saveJson(this.FILE_SEARCH_BY_TITLE, this._searchByTitle);
}
getOutOfDateSearch(): string[] {
return Array.from(this._searchByTitle.entries())
.filter(([title, entry]) => entry.lastUpdateMs + this.MAX_CACHE_AGE_SEARCH_BY_TITLE < Date.now())
.map(([title, entry]) => title);
}
}

53
backend/src/cache/MangaUpdatesCache.ts vendored
View File

@@ -1,20 +1,22 @@
import * as fs from 'fs';
type CacheEntry = { data: string, lastUpdateMs: number }
import {CacheEntry, loadJson, saveJson} from './CacheUtil';
export class MangaUpdatesCache {
private readonly FILE_CACHE_DIR = 'cache';
private readonly FILE_CACHE_DIR = 'cache/mangaupdates';
private readonly FILE_SEARCH_BY_TITLE = this.FILE_CACHE_DIR + '/searchByTitle.json';
private readonly FILE_SERIES_BY_ID = this.FILE_CACHE_DIR + '/seriesById.json';
private readonly FILE_SERIES_GROUPS_BY_ID = this.FILE_CACHE_DIR + '/seriesGroupsById.json';
private readonly FILE_SERIES_IDS_BY_WEBSITE_ID = this.FILE_CACHE_DIR + '/seriesIdsByWebsiteId.json';
private readonly MAX_CACHE_AGE_SEARCH_BY_TITLE = 7 * 24 * 60 * 60 * 1000;
private readonly MAX_CACHE_AGE_SERIES_BY_ID = 30 * 24 * 60 * 60 * 1000;
private readonly MAX_CACHE_AGE_SERIES_GROUPS_BY_ID = 1 * 24 * 60 * 60 * 1000;
private readonly MAX_CACHE_AGE_SERIES_IDS_BY_WEBSITE_ID = 30 * 24 * 60 * 60 * 1000;
private _searchByTitle = new Map<string, CacheEntry>();
private _seriesById = new Map<string, CacheEntry>();
private _seriesGroupsById = new Map<string, CacheEntry>();
private _seriesIdsByWebsiteId = new Map<string, CacheEntry>();
constructor() {
this.load();
@@ -22,32 +24,13 @@ export class MangaUpdatesCache {
private load(): void {
try {
fs.mkdirSync(this.FILE_CACHE_DIR);
fs.mkdirSync(this.FILE_CACHE_DIR, {recursive: true});
} catch (_) {
}
this._searchByTitle = this.loadJson(this.FILE_SEARCH_BY_TITLE);
this._seriesById = this.loadJson(this.FILE_SERIES_BY_ID);
this._seriesGroupsById = this.loadJson(this.FILE_SERIES_GROUPS_BY_ID);
}
private loadJson(file: string): Map<string, CacheEntry> {
try {
const data = fs.readFileSync(file, {encoding: 'utf8', flag: 'r'});
const json = JSON.parse(data);
return new Map<string, CacheEntry>(Object.entries(json));
} catch (err) {
console.error('Failed to load cache from ' + file + ':' + (err as Error).message);
return new Map();
}
}
private saveJson(file: string, map: Map<string, CacheEntry>): void {
try {
const data = JSON.stringify(Object.fromEntries(map.entries()));
fs.writeFileSync(file, data, {encoding: 'utf8', flag: 'w'});
} catch (err) {
console.error(err);
}
this._searchByTitle = loadJson(this.FILE_SEARCH_BY_TITLE);
this._seriesById = loadJson(this.FILE_SERIES_BY_ID);
this._seriesGroupsById = loadJson(this.FILE_SERIES_GROUPS_BY_ID);
this._seriesIdsByWebsiteId = loadJson(this.FILE_SERIES_IDS_BY_WEBSITE_ID);
}
getSearchByTitle(title: string): string | undefined {
@@ -65,19 +48,29 @@ export class MangaUpdatesCache {
return !entry || entry.lastUpdateMs + this.MAX_CACHE_AGE_SERIES_GROUPS_BY_ID < Date.now() ? undefined : entry.data;
}
getSeriesIdByWebsiteId(id: string): string | undefined {
const entry = this._seriesIdsByWebsiteId.get(id);
return !entry || entry.lastUpdateMs + this.MAX_CACHE_AGE_SERIES_IDS_BY_WEBSITE_ID < Date.now() ? undefined : entry.data;
}
putSearchByTitle(title: string, value: string): void {
this._searchByTitle.set(title, {data: value, lastUpdateMs: Date.now()});
this.saveJson(this.FILE_SEARCH_BY_TITLE, this._searchByTitle);
saveJson(this.FILE_SEARCH_BY_TITLE, this._searchByTitle);
}
putSeriesById(id: string, value: string): void {
this._seriesById.set(id, {data: value, lastUpdateMs: Date.now()});
this.saveJson(this.FILE_SERIES_BY_ID, this._seriesById);
saveJson(this.FILE_SERIES_BY_ID, this._seriesById);
}
putSeriesGroupsById(id: string, value: string): void {
this._seriesGroupsById.set(id, {data: value, lastUpdateMs: Date.now()});
this.saveJson(this.FILE_SERIES_GROUPS_BY_ID, this._seriesGroupsById);
saveJson(this.FILE_SERIES_GROUPS_BY_ID, this._seriesGroupsById);
}
putSeriesIdByWebsiteId(id: string, value: string): void {
this._seriesIdsByWebsiteId.set(id, {data: value, lastUpdateMs: Date.now()});
saveJson(this.FILE_SERIES_IDS_BY_WEBSITE_ID, this._seriesIdsByWebsiteId);
}
getOutOfDateSearch(): string[] {