commit 3f1a218ea2dc147cf890b1be755baeb87b37553d Author: Dennis Gunia Date: Thu Jan 8 19:10:51 2026 +0100 initial release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3602361 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +temp \ No newline at end of file diff --git a/api-gateway/.gitignore b/api-gateway/.gitignore new file mode 100644 index 0000000..82f72b6 --- /dev/null +++ b/api-gateway/.gitignore @@ -0,0 +1,4 @@ +dist/* +data/*/*.json +package-lock.json +node_modules \ No newline at end of file diff --git a/api-gateway/data/books.json b/api-gateway/data/books.json new file mode 100644 index 0000000..fd5d49e --- /dev/null +++ b/api-gateway/data/books.json @@ -0,0 +1,77 @@ +{ + "books": [ + "1.Mose", + "2.Mose", + "3.Mose", + "4.Mose", + "5.Mose", + "Josua", + "Richter", + "Rut", + "1.Samuel", + "2.Samuel", + "1.Könige", + "2.Könige", + "1.Chronik", + "2.Chronik", + "Esra", + "Nehemia", + "Ester", + "Hiob", + "Psalm", + "Sprüche", + "Prediger", + "Hoheslied", + "Jesaja", + "Jeremia", + "Klagelieder", + "Hesekiel", + "Daniel", + "Hosea", + "Joel", + "Amos", + "Obadja", + "Jona", + "Micha", + "Nahum", + "Habakuk", + "Zefanja", + "Haggai", + "Sacharja", + "Maleachi", + "Matthäus", + "Markus", + "Lukas", + "Johannes", + "Apostelgeschichte", + "Römer", + "1.Korinther", + "2.Korinther", + "Galater", + "Epheser", + "Philipper", + "Kolosser", + "1.Thessalonicher", + "2.Thessalonicher", + "1.Timotheus", + "2.Timotheus", + "Titus", + "Philemon", + "Hebräer", + "Jakobus", + "1.Petrus", + "2.Petrus", + "1.Johannes", + "2.Johannes", + "3.Johannes", + "Judas", + "Offenbarung", + "Tobit", + "Judit", + "1.Makkabäer", + "2.Makkabäer", + "Weisheit", + "Jesus Sirach", + "Baruch" + ] +} diff --git a/api-gateway/dockerfile b/api-gateway/dockerfile new file mode 100644 index 0000000..0c2d3df --- /dev/null +++ b/api-gateway/dockerfile @@ -0,0 +1,25 @@ +FROM node:16-alpine AS builder +WORKDIR /app + +COPY package*.json ./ +RUN npm ci --only=production && npm cache clean --force + +# Copy source code +COPY tsconfig.json tsconfig.json +COPY src src + +RUN npm install -g typescript +RUN npm install +RUN tsc + + +FROM node:16-alpine +WORKDIR /app + +COPY --from=builder /app/package*.json ./ +COPY --from=builder /app/dist ./dist +COPY data data +RUN npm install + +EXPOSE 8008 +CMD ["node", "dist/index.js"] \ No newline at end of file diff --git a/api-gateway/package.json b/api-gateway/package.json new file mode 100644 index 0000000..827c3e7 --- /dev/null +++ b/api-gateway/package.json @@ -0,0 +1,31 @@ +{ + "name": "api-gateway", + "version": "1.0.0", + "description": "", + "main": "index.js", + "type": "module", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Dennis Gunia", + "license": "ISC", + "devDependencies": { + "@types/express": "^5.0.6", + "@types/node": "^25.0.3", + "eslint": "^9.39.2", + "prettier": "^3.7.4", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + }, + "dependencies": { + "@types/axios": "^0.14.4", + "axios": "^1.13.2", + "dotenv": "^17.2.3", + "express": "^5.2.1", + "helmet": "^8.1.0", + "http": "^0.0.1-security", + "https": "^1.0.0", + "node-cache": "^5.1.2", + "node-html-parser": "^7.0.2" + } +} diff --git a/api-gateway/src/config/config.ts b/api-gateway/src/config/config.ts new file mode 100644 index 0000000..9f541f7 --- /dev/null +++ b/api-gateway/src/config/config.ts @@ -0,0 +1,19 @@ +import dotenv from 'dotenv'; + +dotenv.config(); + +interface Config { + port: number; + nodeEnv: string; + enableDownload: boolean; + downloadDelay: number; +} + +const config: Config = { + port: Number(process.env.PORT) || 8008, + nodeEnv: process.env.NODE_ENV || 'development', + enableDownload: process.env.ENABLE_DOWNLOAD === 'true' || false, + downloadDelay: parseInt(process.env.DELAY_DOWNLOAD || '15'), +}; + +export default config; \ No newline at end of file diff --git a/api-gateway/src/controllers/book.controller.ts b/api-gateway/src/controllers/book.controller.ts new file mode 100644 index 0000000..5c8d370 --- /dev/null +++ b/api-gateway/src/controllers/book.controller.ts @@ -0,0 +1,62 @@ + +import { Request, Response, NextFunction } from 'express'; +import { BibleProvider } from '../providers/bible.provider.js'; +import config from '../config/config.js'; + +export namespace LibraryController { + export const getTranslationJSON = (req: Request, res: Response, next: NextFunction) => { + let data: any = BibleProvider.get_books(req.params.translation); + if (!data) { + res.json({ + requested_at: new Date().toISOString(), + rescource: "translation", + data: { + message: `translation ${req.params.translation} not found` + }, + success: false + }); + } else { + res.json({ + requested_at: new Date().toISOString(), + rescource: "translation", + data: { + translation: req.params.translation, + books: data + }, + success: true + }); + } + } + + export const listTranslationsJSON = (req: Request, res: Response, next: NextFunction) => { + res.json({ + requested_at: new Date().toISOString(), + rescource: "translations", + data: { + translations: BibleProvider.list_translations() + }, + success: true + }); + } + + export const pullTranslation = (req: Request, res: Response, next: NextFunction) => { + if (config.enableDownload) { + res.json({ + requested_at: new Date().toISOString(), + rescource: "translations", + data: BibleProvider.start_download(req.params.translation), + success: true + }); + }else{ + res.json({ + requested_at: new Date().toISOString(), + rescource: "translations", + data: { + message: "Downloads not enabled. Contact system administrator!" + }, + success: false + }); + } + + } +} \ No newline at end of file diff --git a/api-gateway/src/controllers/generic.controller.ts b/api-gateway/src/controllers/generic.controller.ts new file mode 100644 index 0000000..768ecba --- /dev/null +++ b/api-gateway/src/controllers/generic.controller.ts @@ -0,0 +1,14 @@ + +import { Request, Response, NextFunction } from 'express'; +import { BibleProvider } from '../providers/bible.provider.js'; + +export namespace GenericController { + export const cacheStats = (req: Request, res: Response, next: NextFunction) => { + res.json({ + requested_at: new Date().toISOString(), + rescource: "stats_cache", + data: BibleProvider.get_cache_metrics(), + success: true + }); + } +} \ No newline at end of file diff --git a/api-gateway/src/controllers/speech.controller.ts b/api-gateway/src/controllers/speech.controller.ts new file mode 100644 index 0000000..b04feff --- /dev/null +++ b/api-gateway/src/controllers/speech.controller.ts @@ -0,0 +1,148 @@ + +import { Request, Response, NextFunction } from 'express'; +import { BibleProvider } from '../providers/bible.provider.js'; + +export namespace SpeechController { + export const translateText = (req: Request, res: Response, next: NextFunction) => { + const binaryString = atob(req.params.inputstring); + const uint8Array = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + uint8Array[i] = binaryString.charCodeAt(i); // Get byte value of each character + } + const input = new TextDecoder('utf-8').decode(uint8Array).toLowerCase() + .replace("-", " bis ") + .replace("lucas", "lukas") + .replace("jacobus", "jakobus") + .replace("chroniken", "chronik") + .replace("phillipa", "philipper") + .replace("esther", "ester") + .replace("hey seal", "hesekiel") + .replace("hohes lied", "hoheslied") + .replace("hop", "hiob") + .replace("hagi", "haggai") + .replace("thessalonich", "thessalonicher") + .replace("joshua", "josua") + .replace("moose", "mose") + .replace("malachi", "maleachi") + .replace("kolosse", "kolosser") + .replace("sacha", "sacharja") + .replace("n ahum", "nahum") + .replace("nach um", "nahum"); + console.log(input) + + // list of books to compare to + const regx1 = new RegExp("^[0-9]\.", "g"); + const books_vf1 = BibleProvider.get_all_book_names()?.map(el => el.toLowerCase().replace(regx1, "")) || []; + let book = ""; + let segments: string[] = []; + + for (let cbook of books_vf1) { + let ix = input.indexOf(cbook); + //console.log(cbook, ix) + + if (ix == -1) continue; + book = cbook; + segments = input.split(cbook) + } + console.log(book) + if (segments.length == 2) { + const prefix_raw = segments[0].trim(); + const prefix_new = prefix_raw + .replace("erster", "1.") + .replace("erste", "1.") + .replace("zweite", "2.") + .replace("dritte", "3.") + .replace("vierte", "4.") + .replace("fünfte", "5.") + book = prefix_new + book.charAt(0).toUpperCase() + String(book).slice(1); + } + console.log(book) + // verify book exists + const books_vf2 = BibleProvider.get_all_book_names() || []; + if (!books_vf2.includes(book)) { + res.json("") + return + } + + // split chapter and verse + let numeric_parts: string[] = [] + let segment_numeric = "" + + segment_numeric = (segments.length == 2) ? segments[1]: segments[0]; + if (segment_numeric.indexOf("vers") > -1){ + numeric_parts = segment_numeric.split("vers").map(el => el.trim()); + }else if (segment_numeric.trim().indexOf(" ") > -1){ + numeric_parts = segment_numeric.trim().split(" ").map(el => el.trim()); + }else{ + segment_numeric += " vers 1 bis 1000"; + numeric_parts = segment_numeric.trim().split("vers").map(el => el.trim()); + } + + console.log(book, numeric_parts) + + const txttonum_single = (inp: string): number => { + switch (inp) { + case "eins": return 1; + case "ein": return 1; + case "zwei": return 2; + case "drei": return 3; + case "vier": return 4; + case "fünf": return 5; + case "sechs": return 6; + case "sieben": return 7; + case "acht": return 8; + case "neun": return 9; + case "zehn": return 10; + case "zwanzig": return 20; + case "dreißig": return 30; + case "vierzig": return 40; + case "fünfzig": return 50; + case "sechzig": return 60; + case "siebzig": return 70; + case "achtzig": return 80; + case "neunzig": return 90; + case "hundert": return 1; + default: return 0 + } + } + const conv_str = (input: string) => { + let result = parseInt(input); + if (Number.isNaN(result)) { + result = 0; + } else { + return result; + } + if (input.indexOf("hundert") >= 0) { + let i1 = input.split("hundert")[0]; + input = input.split("hundert")[1]; + let i2 = txttonum_single(i1) * 100; + result += i2 > 0 ? i2 : 100 + } + if (input == "elf") { + result += 11; + } else if (input == "zwölf") { + result += 12; + } else if (input.indexOf("zehn") >= 0) { + let i1 = txttonum_single(input.replace("zehn", "")) + result += i1 + 10; + } else if (input.indexOf("und") >= 0) { + let i1 = input.split("und")[0]; + input = input.split("und")[1]; + result += txttonum_single(i1); + result += txttonum_single(input); + } else { + result += txttonum_single(input); + } + return result; + + } + + let verse_str = numeric_parts[1].split("bis").map(el => el.trim()) + if (verse_str.length == 1){ + res.json(`${book}/${conv_str(numeric_parts[0])}/${conv_str(verse_str[0])}`); + }else{ + res.json(`${book}/${conv_str(numeric_parts[0])}/${conv_str(verse_str[0])}-${conv_str(verse_str[1])}`); + + } + } +} \ No newline at end of file diff --git a/api-gateway/src/controllers/translation.controller.ts b/api-gateway/src/controllers/translation.controller.ts new file mode 100644 index 0000000..2b2803f --- /dev/null +++ b/api-gateway/src/controllers/translation.controller.ts @@ -0,0 +1,124 @@ + +import { Request, Response, NextFunction } from 'express'; +import { BibleProvider } from '../providers/bible.provider.js'; +import { request } from 'node:http'; + +export namespace TranslationController { + export const getBookJSON = (req: Request, res: Response, next: NextFunction) => { + let data = BibleProvider.get_book(req.params.translation, req.params.book); + + if (!data) { + res.json({ + requested_at: new Date().toISOString(), + rescource: "book", + data: { + message: `book ${req.params.book} not found in ${req.params.translation}` + }, + success: false + }); + } else { + res.json({ + requested_at: new Date().toISOString(), + rescource: "book", + data: { + translation: req.params.translation, + book: req.params.book, + chapters: data + }, + success: true + }); + } + } + + export const getChapterJSON = (req: Request, res: Response, next: NextFunction) => { + let data = BibleProvider.get_chapter(req.params.translation, req.params.book, parseInt(req.params.chapter)); + if (!data || data.length === 0) { + res.json({ + requested_at: new Date().toISOString(), + rescource: "chapter", + data: { + message: `chapter ${req.params.chapter} of book ${req.params.book} not found in ${req.params.translation}` + }, + success: false + }); + } else { + res.json({ + requested_at: new Date().toISOString(), + rescource: "chapter", + data: { + translation: req.params.translation, + book: req.params.book, + chapter: req.params.chapter, + verses: data + }, + success: true + }); + } + + } + + export const getVerseJSON = (req: Request, res: Response, next: NextFunction) => { + let data = "" + let verses = []; + if (req.params.verse.includes("-")) { + // handle range + let ranges = req.params.verse.split('-'); + for (let v = parseInt(ranges[0]); v <= parseInt(ranges[1]); v++) { + let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), v); + if (verse_obj) { + verses.push(verse_obj) + data += `${verse_obj.text} `; + } + } + + } else { + let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), parseInt(req.params.verse)); + if (verse_obj) { + verses.push(verse_obj) + data = verse_obj.text; + } + } + + res.json({ + requested_at: new Date().toISOString(), + rescource: "verse", + data: { + translation: req.params.translation, + book: req.params.book, + chapter: req.params.chapter, + verse: req.params.verse, + text: data.trimEnd(), + verses: verses + }, + success: true + }); + + } + + export const getVerseRaw = (req: Request, res: Response, next: NextFunction) => { + let data = "" + if (req.params.verse.includes("-")) { + // handle range + let ranges = req.params.verse.split('-'); + for (let v = parseInt(ranges[0]); v <= parseInt(ranges[1]); v++) { + let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), v); + if (verse_obj) { + data += `${verse_obj.text} `; + } + } + } else { + let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), parseInt(req.params.verse)); + if (verse_obj) { + data = verse_obj.text; + } + } + console.log("test") + if (!data) { + res.status(404).json(`Kapitel ${req.params.chapter} vers ${req.params.verse} vom buch ${req.params.book} konnte in der Übersetzung ${req.params.translation} nicht gefunden werden.`); + } else { + res.charset = "UTF-8" + res.json(data); + } + } + +} \ No newline at end of file diff --git a/api-gateway/src/index.ts b/api-gateway/src/index.ts new file mode 100644 index 0000000..93b3903 --- /dev/null +++ b/api-gateway/src/index.ts @@ -0,0 +1,61 @@ +import { BibleProvider } from "./providers/bible.provider.js"; +import express from 'express'; +import defaultRouter from "./routes/default.routes.js"; +import config from "./config/config.js"; +import helmet from "helmet"; +import { exit } from "process"; +import { DownloaderProvider } from "./providers/downloader.provider.js"; + +//BibleProvider.load_biliothek(); +//console.log(`Loaded ${BibleProvider.biliothek.length} translations in biliothek`); + + +//const downloader = new DownloaderProvider.Downloader('SLT'); +//downloader.start(); + +const app = express(); +app.use(express.json()); +app.use(helmet()) +app.disable('x-powered-by') + +app.use((req: express.Request, res: express.Response, next: express.NextFunction) => { + // log accessed url + // calculate processing time + const start = Date.now(); + next(); + const duration = Date.now() - start; + const access_log = { + date: new Date().toISOString(), + method: req.method, + url: req.url, + ip: req.ip, + agent: req.headers['user-agent'] || '', + status: res.statusCode, + duration: duration + } + console.log(access_log); + + +}); + +app.use('/', defaultRouter); + + + +app.use((err: any, req: express.Request, res: express.Response, next: express.NextFunction) => { + console.error(err.stack); + res.status(500).json({ + requested_at: new Date().toISOString(), + rescource: "error", + data: { + message: 'Internal Server Error' + }, + success: false + }); +}); + + +app.listen(config.port, () => { + console.log(`Server running on port ${config.port}`); +}); + diff --git a/api-gateway/src/providers/bible.provider.ts b/api-gateway/src/providers/bible.provider.ts new file mode 100644 index 0000000..eccb164 --- /dev/null +++ b/api-gateway/src/providers/bible.provider.ts @@ -0,0 +1,183 @@ +import * as fs from 'fs'; +import NodeCache from 'node-cache'; +import { DownloaderProvider } from './downloader.provider.js'; + +export namespace BibleProvider { + + let cache = new NodeCache(); + + export interface Translation { + translation: string; + books: Book[]; + } + + export interface Book { + book: string; + chapters: Chapter[]; + } + + export interface Chapter { + translation?: string; + book?: string; + chapter?: number; + verse: number; + text: string; + } + + setInterval(() => { + let completed_downloads = downloads.filter(el => el.getStatus() == "completed" || el.getStatus() == "error") + for (let completed_download of completed_downloads) { + if (completed_download.time_end) { + const time = new Date(); + const time_since = (time.getTime() - completed_download.time_end.getTime()) / 1000; + if (time_since > 60 * 15) { + const index = downloads.indexOf(completed_download) + if (index > -1) { // only splice array when item is found + downloads.splice(index, 1); + } + console.log("Removed old download item") + } + } + } + }, 60000) // called every minute + + let downloads: DownloaderProvider.Downloader[] = [] + + export const get_all_book_names = () => { + let books: string[] | undefined = cache.get(`books`); + if (books == undefined) { + const book_list_file = `data/books.json`; + books = JSON.parse(fs.readFileSync(book_list_file, 'utf-8')).books; + cache.set('books', books, 60000) + } + return books; + } + + export const start_download = (translation: string) => { + let existing_download = downloads.filter((el: DownloaderProvider.Downloader) => el.translation == translation); + if (existing_download.length > 0) { + return { + status: existing_download[0].getStatus(), + uuid: existing_download[0].uuid, + logs: existing_download[0].logs + } + } else { + let downloader = new DownloaderProvider.Downloader(translation) + downloads.push(downloader); + downloader.start(); + return { + status: downloader.getStatus(), + uuid: downloader.uuid, + logs: downloader.logs + } + } + + } + + export const list_translations = (): string[] => { + let translations: string[] | undefined = cache.get('translations') + if (translations == undefined) { + console.log("Translations not in cache. Load from disk") + // load translations from disk + translations = [] + let folders = fs.readdirSync('data', { withFileTypes: true }) + for (let folder of folders) { + if (folder.isDirectory()) { + translations.push(folder.name) + } + } + cache.set('translations', translations, 300) + } + return translations; + } + + export const get_translation = (translation: string): Translation | undefined => { + let translation_obj: Translation | undefined = cache.get(`translation:${translation}`); + if (translation_obj == undefined) { + console.log("Translation not in cache. Load from disk") + + translation_obj = { + translation: translation, + books: [] + } + // load book names + let files = fs.readdirSync(`data/${translation}`, { withFileTypes: true }); + for (let file of files) { + if (file.isFile() && file.name.endsWith('.json')) { + let book_name = file.name.replace('.json', ''); + translation_obj.books.push({ + book: book_name, + chapters: [] + }) + } + } + cache.set(`translation:${translation}`, translation_obj, 300) + } + return translation_obj; + } + + export const get_books = (translation: string): string[] | undefined => { + let t = get_translation(translation); + if (!t) return undefined; + return t.books.map(b => b.book); + } + + export const get_book = (translation: string, book: string) => { + let t = get_translation(translation); + if (!t) return undefined; + let book_obj: Book | undefined = cache.get(`translation:${translation}:${book}`) + if (book_obj == undefined) { + book_obj = { + book: book, + chapters: [] + } + console.log("Book not in cache. Load from disk") + try { + let book_file = `${book}.json` + let book_content = JSON.parse(fs.readFileSync(`data/${translation}/${book_file}`, 'utf-8')); + if (!book_content || book_content.length === 0) { + cache.set(`translation:${translation}:${book}`, null, 300) // set nul to mark that it does not exist + return undefined; + } + book_obj.chapters = book_content.map((c: any) => { + return { chapter: c.chapter, verse: c.verse, text: c.text }; + }); + cache.set(`translation:${translation}:${book}`, book_obj, 300) + + } catch (error) { + console.error(error) + return undefined; + } + } + return book_obj?.chapters; + } + + export const get_chapter = (translation: string, book: string, chapter: number) => { + let chapter_obj: Chapter[] | undefined = cache.get(`translation:${translation}:${book}:${chapter}`) + if (chapter_obj == undefined) { + console.log("Chapter not in cache. Generate data.") + let b = get_book(translation, book); + if (!b) { + cache.set(`translation:${translation}:${book}:${chapter}`, null, 300); + return undefined + } + chapter_obj = b.filter(c => c.chapter === chapter).map(c => { + return { verse: c.verse, text: c.text }; + }); + cache.set(`translation:${translation}:${book}:${chapter}`, chapter_obj, 300); + + } + return chapter_obj; + } + + + export const get_verse = (translation: string, book: string, chapter: number, verse: number): Chapter | undefined => { + let chapters = get_chapter(translation, book, chapter); + if (!chapters) return undefined; + return chapters.find(v => v.verse === verse); + } + + export const get_cache_metrics = () => { + return cache.getStats(); + } +} \ No newline at end of file diff --git a/api-gateway/src/providers/downloader.provider.ts b/api-gateway/src/providers/downloader.provider.ts new file mode 100644 index 0000000..3d5b900 --- /dev/null +++ b/api-gateway/src/providers/downloader.provider.ts @@ -0,0 +1,185 @@ +import axios from 'axios'; +import { randomUUID } from 'crypto'; +import * as fs from 'fs'; +import { parse } from 'node-html-parser'; +import config from '../config/config.js'; + +export namespace DownloaderProvider { + const bibleserver_endpoint = 'https://www.bibleserver.com'; + + export class Downloader { + private _translation: string; + private _logs: string[] = []; + private _status: 'idle' | 'running' | 'completed' | 'error' = 'idle'; + private _books: string[] = []; + private _data_directory: string = 'data'; + private _operation_id: string = '' + private _time_start: Date | undefined; + private _time_end: Date | undefined; + + public get translation() { + return this._translation + } + + public get uuid() { + return this._operation_id + } + + public get logs() { + return this._logs + } + + public get time_start() { + return this._time_start + } + + public get time_end() { + return this._time_end + } + + constructor(private translation_in: string) { + this._translation = translation_in; + this._operation_id = randomUUID() + this._status = 'idle'; + this.log(`Initialized downloader for translation: ${this._translation}`); + this.log(`Using bibleserver endpoint: ${bibleserver_endpoint}`); + + } + + public start() { + this._time_start = new Date(); + this.log(`Starting download for translation: ${this._translation}`); + this._status = 'running'; + // get all translations from reference file + try { + const book_list_file = `${this._data_directory}/books.json`; + this.log(`Loading book list from ${book_list_file}`); + this._books = JSON.parse(fs.readFileSync(book_list_file, 'utf-8')).books; + this.log(`Loaded ${this._books.length} books to download`); + } catch (error) { + this.log(`Error loading book list: ${error}`); + this._status = 'error'; + return; + } + // create directory + try { + if (!fs.existsSync(`${this._data_directory}/${this._translation}`)) { + fs.mkdirSync(`${this._data_directory}/${this._translation}`, { recursive: true }); + this.log(`Created directory: ${this._data_directory}/${this._translation}`); + } + } catch (error) { + this.log(`Error creating translation directory: ${error}`); + this._status = 'error'; + return; + } + + this.fetch_all_books(); + + } + + public getStatus(): 'idle' | 'running' | 'completed' | 'error' { + return this._status; + } + + private delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + private async fetch_all_books() { + try { + for (let book of this._books) { + if (fs.existsSync(`${this._data_directory}/${this._translation}/${book}.json`)) { + this.log(`Book ${book} already exists for ${this._translation}, skipping`); + continue; + } + this.log(`Fetching book: ${book} for translation: ${this._translation}`); + + let chapter = 1; + let book_content: any[] = []; + + while (true) { + this.log(`Trying to fetch chapter ${chapter} of book ${book}`); + await this.delay(config.downloadDelay * 1000); // sleep to prevent DDOS + let chapter_content: any = await this.fetch_chapter(book, chapter); + if (chapter_content.length === 0) { + break; + } + book_content = book_content.concat(chapter_content); + chapter++; + } + fs.writeFileSync(`${this._data_directory}/${this._translation}/${book}.json`, JSON.stringify(book_content, null, 4)); + this.log(`Saved book ${book} for translation ${this._translation} with ${book_content.length} verses`); + this.log(`Completed fetching book: ${book} for translation: ${this._translation}`); + } + this._time_end = new Date(); + this._status = "completed" + } catch (error) { + this.log(`Error fetching books: ${error}`); + this._status = 'error'; + return; + } + } + + private async fetch_chapter(book: string, chapter: number) { + let bibleserver_url = `${bibleserver_endpoint}/${this._translation}/${book}${chapter}`; + this.log(`Fetching URL from: ${bibleserver_url}`) + try { + let response = await axios.get(bibleserver_url) + this.log("Received response") + let html = response.data; + let root = parse(html); + // verify to avoid redirect + let book_verify_name = root.querySelector('.chapter')?.querySelector('header')?.querySelector('h1')?.text.trim() || ''; + if (book_verify_name !== `${book} ${chapter}`) { + // chapter does not exist, return empty list + return []; + } + let verse_elements = root.querySelectorAll('.verse'); + let result_array = []; + for (let verse_element of verse_elements) { + verse_element.querySelectorAll('.footnote').forEach(fn => fn.remove()); // remove footnotes + let verse_raw = verse_element.querySelector('.verse-number')?.childNodes[0].text + // resolve verse ranges + if (verse_raw?.includes('-')) { + let ranges = verse_raw.split('-'); + for (let v = parseInt(ranges[0]); v <= parseInt(ranges[1]); v++) { + result_array.push({ + translation: this._translation, + book: book, + chapter: chapter, + verse: v, + text: verse_element.querySelector('.verse-content')?.childNodes[0].text || '' + }); + } + } else { + result_array.push({ + translation: this._translation, + book: book, + chapter: chapter, + verse: Number(verse_element.querySelector('.verse-number')?.childNodes[0].text) || -1, + text: verse_element.querySelector('.verse-content')?.childNodes[0].text || '' + }); + } + } + return result_array + } catch (error: any) { + if (error.response && error.response.status === 404) { + // translation does not exist + this.log(`Translation ${this._translation} does not exist for book ${book}`); + return []; + } else { + this.log(`Error fetching ${this._translation} ${book} ${chapter}- ${error}`); + this.log(`URL: ${bibleserver_url}`); + throw (error) + } + } + } + + private log(message: string) { + let log_entry = `[Downloader<${this._operation_id}>:${this._translation}][${new Date().toISOString()}] ${message}`; + this._logs.push(log_entry); + console.log(log_entry); + } + + } +} \ No newline at end of file diff --git a/api-gateway/src/routes/default.routes.ts b/api-gateway/src/routes/default.routes.ts new file mode 100644 index 0000000..de90ffa --- /dev/null +++ b/api-gateway/src/routes/default.routes.ts @@ -0,0 +1,33 @@ +import { NextFunction, Router, Request, Response } from "express"; +import { TranslationController } from "../controllers/translation.controller.js"; +import { LibraryController } from "../controllers/book.controller.js"; +import { GenericController } from "../controllers/generic.controller.js"; +import { SpeechController } from "../controllers/speech.controller.js"; + +const defaultRouter = Router() + +defaultRouter.get('/_statsCache',GenericController.cacheStats); +defaultRouter.get('/_convert/:inputstring',SpeechController.translateText); + + +defaultRouter.get('/', LibraryController.listTranslationsJSON); +defaultRouter.get('/:translation', LibraryController.getTranslationJSON); +defaultRouter.get('/:translation/_pull', LibraryController.pullTranslation); +defaultRouter.get('/:translation/:book', TranslationController.getBookJSON); +defaultRouter.get('/:translation/:book/:chapter', TranslationController.getChapterJSON); +defaultRouter.get('/:translation/:book/:chapter/:verse', TranslationController.getVerseJSON); +defaultRouter.get('/:translation/:book/:chapter/:verse/raw', TranslationController.getVerseRaw); + + +defaultRouter.use((err: any, req: Request, res: Response, next: NextFunction) => { + console.log(err) + res.status(404).json({ + requested_at: new Date().toISOString(), + rescource: "error", + data: { + message: 'Resource not found' + }, + success: false + }); +}); +export default defaultRouter; \ No newline at end of file diff --git a/api-gateway/tsconfig.json b/api-gateway/tsconfig.json new file mode 100644 index 0000000..dd08f18 --- /dev/null +++ b/api-gateway/tsconfig.json @@ -0,0 +1,38 @@ +{ + // Visit https://aka.ms/tsconfig to read more about this file + "compilerOptions": { + // File Layout + "rootDir": "./src", + "outDir": "./dist", + // Environment Settings + // See also https://aka.ms/tsconfig/module + "module": "nodenext", + "target": "esnext", + "moduleResolution": "nodenext", + + // For nodejs: + // "lib": ["esnext"], + // "types": ["node"], + // and npm install -D @types/node + // Other Outputs + "sourceMap": true, + "declaration": true, + "declarationMap": true, + // Stricter Typechecking Options + "exactOptionalPropertyTypes": true, + // Style Options + // "noImplicitReturns": true, + // "noImplicitOverride": true, + // "noUnusedLocals": true, + // "noUnusedParameters": true, + // "noFallthroughCasesInSwitch": true, + // "noPropertyAccessFromIndexSignature": true, + // Recommended Options + "strict": true, + "jsx": "react-jsx", + "verbatimModuleSyntax": false, + "isolatedModules": true, + "moduleDetection": "force", + "skipLibCheck": true, + } +} \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..2892851 --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd api-gateway +sudo docker build . -t registry.dennisgunia.de/bibleapi:latest +sudo docker push registry.dennisgunia.de/bibleapi:latest