initial release
This commit is contained in:
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
temp
|
||||
4
api-gateway/.gitignore
vendored
Normal file
4
api-gateway/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
dist/*
|
||||
data/*/*.json
|
||||
package-lock.json
|
||||
node_modules
|
||||
77
api-gateway/data/books.json
Normal file
77
api-gateway/data/books.json
Normal file
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"books": [
|
||||
"1.Mose",
|
||||
"2.Mose",
|
||||
"3.Mose",
|
||||
"4.Mose",
|
||||
"5.Mose",
|
||||
"Josua",
|
||||
"Richter",
|
||||
"Rut",
|
||||
"1.Samuel",
|
||||
"2.Samuel",
|
||||
"1.Könige",
|
||||
"2.Könige",
|
||||
"1.Chronik",
|
||||
"2.Chronik",
|
||||
"Esra",
|
||||
"Nehemia",
|
||||
"Ester",
|
||||
"Hiob",
|
||||
"Psalm",
|
||||
"Sprüche",
|
||||
"Prediger",
|
||||
"Hoheslied",
|
||||
"Jesaja",
|
||||
"Jeremia",
|
||||
"Klagelieder",
|
||||
"Hesekiel",
|
||||
"Daniel",
|
||||
"Hosea",
|
||||
"Joel",
|
||||
"Amos",
|
||||
"Obadja",
|
||||
"Jona",
|
||||
"Micha",
|
||||
"Nahum",
|
||||
"Habakuk",
|
||||
"Zefanja",
|
||||
"Haggai",
|
||||
"Sacharja",
|
||||
"Maleachi",
|
||||
"Matthäus",
|
||||
"Markus",
|
||||
"Lukas",
|
||||
"Johannes",
|
||||
"Apostelgeschichte",
|
||||
"Römer",
|
||||
"1.Korinther",
|
||||
"2.Korinther",
|
||||
"Galater",
|
||||
"Epheser",
|
||||
"Philipper",
|
||||
"Kolosser",
|
||||
"1.Thessalonicher",
|
||||
"2.Thessalonicher",
|
||||
"1.Timotheus",
|
||||
"2.Timotheus",
|
||||
"Titus",
|
||||
"Philemon",
|
||||
"Hebräer",
|
||||
"Jakobus",
|
||||
"1.Petrus",
|
||||
"2.Petrus",
|
||||
"1.Johannes",
|
||||
"2.Johannes",
|
||||
"3.Johannes",
|
||||
"Judas",
|
||||
"Offenbarung",
|
||||
"Tobit",
|
||||
"Judit",
|
||||
"1.Makkabäer",
|
||||
"2.Makkabäer",
|
||||
"Weisheit",
|
||||
"Jesus Sirach",
|
||||
"Baruch"
|
||||
]
|
||||
}
|
||||
25
api-gateway/dockerfile
Normal file
25
api-gateway/dockerfile
Normal file
@@ -0,0 +1,25 @@
|
||||
FROM node:16-alpine AS builder
|
||||
WORKDIR /app
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm ci --only=production && npm cache clean --force
|
||||
|
||||
# Copy source code
|
||||
COPY tsconfig.json tsconfig.json
|
||||
COPY src src
|
||||
|
||||
RUN npm install -g typescript
|
||||
RUN npm install
|
||||
RUN tsc
|
||||
|
||||
|
||||
FROM node:16-alpine
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /app/package*.json ./
|
||||
COPY --from=builder /app/dist ./dist
|
||||
COPY data data
|
||||
RUN npm install
|
||||
|
||||
EXPOSE 8008
|
||||
CMD ["node", "dist/index.js"]
|
||||
31
api-gateway/package.json
Normal file
31
api-gateway/package.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "api-gateway",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "Dennis Gunia",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/node": "^25.0.3",
|
||||
"eslint": "^9.39.2",
|
||||
"prettier": "^3.7.4",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/axios": "^0.14.4",
|
||||
"axios": "^1.13.2",
|
||||
"dotenv": "^17.2.3",
|
||||
"express": "^5.2.1",
|
||||
"helmet": "^8.1.0",
|
||||
"http": "^0.0.1-security",
|
||||
"https": "^1.0.0",
|
||||
"node-cache": "^5.1.2",
|
||||
"node-html-parser": "^7.0.2"
|
||||
}
|
||||
}
|
||||
19
api-gateway/src/config/config.ts
Normal file
19
api-gateway/src/config/config.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
interface Config {
|
||||
port: number;
|
||||
nodeEnv: string;
|
||||
enableDownload: boolean;
|
||||
downloadDelay: number;
|
||||
}
|
||||
|
||||
const config: Config = {
|
||||
port: Number(process.env.PORT) || 8008,
|
||||
nodeEnv: process.env.NODE_ENV || 'development',
|
||||
enableDownload: process.env.ENABLE_DOWNLOAD === 'true' || false,
|
||||
downloadDelay: parseInt(process.env.DELAY_DOWNLOAD || '15'),
|
||||
};
|
||||
|
||||
export default config;
|
||||
62
api-gateway/src/controllers/book.controller.ts
Normal file
62
api-gateway/src/controllers/book.controller.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
import { BibleProvider } from '../providers/bible.provider.js';
|
||||
import config from '../config/config.js';
|
||||
|
||||
export namespace LibraryController {
|
||||
export const getTranslationJSON = (req: Request, res: Response, next: NextFunction) => {
|
||||
let data: any = BibleProvider.get_books(req.params.translation);
|
||||
if (!data) {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "translation",
|
||||
data: {
|
||||
message: `translation ${req.params.translation} not found`
|
||||
},
|
||||
success: false
|
||||
});
|
||||
} else {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "translation",
|
||||
data: {
|
||||
translation: req.params.translation,
|
||||
books: data
|
||||
},
|
||||
success: true
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const listTranslationsJSON = (req: Request, res: Response, next: NextFunction) => {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "translations",
|
||||
data: {
|
||||
translations: BibleProvider.list_translations()
|
||||
},
|
||||
success: true
|
||||
});
|
||||
}
|
||||
|
||||
export const pullTranslation = (req: Request, res: Response, next: NextFunction) => {
|
||||
if (config.enableDownload) {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "translations",
|
||||
data: BibleProvider.start_download(req.params.translation),
|
||||
success: true
|
||||
});
|
||||
}else{
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "translations",
|
||||
data: {
|
||||
message: "Downloads not enabled. Contact system administrator!"
|
||||
},
|
||||
success: false
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
14
api-gateway/src/controllers/generic.controller.ts
Normal file
14
api-gateway/src/controllers/generic.controller.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
import { BibleProvider } from '../providers/bible.provider.js';
|
||||
|
||||
export namespace GenericController {
|
||||
export const cacheStats = (req: Request, res: Response, next: NextFunction) => {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "stats_cache",
|
||||
data: BibleProvider.get_cache_metrics(),
|
||||
success: true
|
||||
});
|
||||
}
|
||||
}
|
||||
148
api-gateway/src/controllers/speech.controller.ts
Normal file
148
api-gateway/src/controllers/speech.controller.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
import { BibleProvider } from '../providers/bible.provider.js';
|
||||
|
||||
export namespace SpeechController {
|
||||
export const translateText = (req: Request, res: Response, next: NextFunction) => {
|
||||
const binaryString = atob(req.params.inputstring);
|
||||
const uint8Array = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
uint8Array[i] = binaryString.charCodeAt(i); // Get byte value of each character
|
||||
}
|
||||
const input = new TextDecoder('utf-8').decode(uint8Array).toLowerCase()
|
||||
.replace("-", " bis ")
|
||||
.replace("lucas", "lukas")
|
||||
.replace("jacobus", "jakobus")
|
||||
.replace("chroniken", "chronik")
|
||||
.replace("phillipa", "philipper")
|
||||
.replace("esther", "ester")
|
||||
.replace("hey seal", "hesekiel")
|
||||
.replace("hohes lied", "hoheslied")
|
||||
.replace("hop", "hiob")
|
||||
.replace("hagi", "haggai")
|
||||
.replace("thessalonich", "thessalonicher")
|
||||
.replace("joshua", "josua")
|
||||
.replace("moose", "mose")
|
||||
.replace("malachi", "maleachi")
|
||||
.replace("kolosse", "kolosser")
|
||||
.replace("sacha", "sacharja")
|
||||
.replace("n ahum", "nahum")
|
||||
.replace("nach um", "nahum");
|
||||
console.log(input)
|
||||
|
||||
// list of books to compare to
|
||||
const regx1 = new RegExp("^[0-9]\.", "g");
|
||||
const books_vf1 = BibleProvider.get_all_book_names()?.map(el => el.toLowerCase().replace(regx1, "")) || [];
|
||||
let book = "";
|
||||
let segments: string[] = [];
|
||||
|
||||
for (let cbook of books_vf1) {
|
||||
let ix = input.indexOf(cbook);
|
||||
//console.log(cbook, ix)
|
||||
|
||||
if (ix == -1) continue;
|
||||
book = cbook;
|
||||
segments = input.split(cbook)
|
||||
}
|
||||
console.log(book)
|
||||
if (segments.length == 2) {
|
||||
const prefix_raw = segments[0].trim();
|
||||
const prefix_new = prefix_raw
|
||||
.replace("erster", "1.")
|
||||
.replace("erste", "1.")
|
||||
.replace("zweite", "2.")
|
||||
.replace("dritte", "3.")
|
||||
.replace("vierte", "4.")
|
||||
.replace("fünfte", "5.")
|
||||
book = prefix_new + book.charAt(0).toUpperCase() + String(book).slice(1);
|
||||
}
|
||||
console.log(book)
|
||||
// verify book exists
|
||||
const books_vf2 = BibleProvider.get_all_book_names() || [];
|
||||
if (!books_vf2.includes(book)) {
|
||||
res.json("")
|
||||
return
|
||||
}
|
||||
|
||||
// split chapter and verse
|
||||
let numeric_parts: string[] = []
|
||||
let segment_numeric = ""
|
||||
|
||||
segment_numeric = (segments.length == 2) ? segments[1]: segments[0];
|
||||
if (segment_numeric.indexOf("vers") > -1){
|
||||
numeric_parts = segment_numeric.split("vers").map(el => el.trim());
|
||||
}else if (segment_numeric.trim().indexOf(" ") > -1){
|
||||
numeric_parts = segment_numeric.trim().split(" ").map(el => el.trim());
|
||||
}else{
|
||||
segment_numeric += " vers 1 bis 1000";
|
||||
numeric_parts = segment_numeric.trim().split("vers").map(el => el.trim());
|
||||
}
|
||||
|
||||
console.log(book, numeric_parts)
|
||||
|
||||
const txttonum_single = (inp: string): number => {
|
||||
switch (inp) {
|
||||
case "eins": return 1;
|
||||
case "ein": return 1;
|
||||
case "zwei": return 2;
|
||||
case "drei": return 3;
|
||||
case "vier": return 4;
|
||||
case "fünf": return 5;
|
||||
case "sechs": return 6;
|
||||
case "sieben": return 7;
|
||||
case "acht": return 8;
|
||||
case "neun": return 9;
|
||||
case "zehn": return 10;
|
||||
case "zwanzig": return 20;
|
||||
case "dreißig": return 30;
|
||||
case "vierzig": return 40;
|
||||
case "fünfzig": return 50;
|
||||
case "sechzig": return 60;
|
||||
case "siebzig": return 70;
|
||||
case "achtzig": return 80;
|
||||
case "neunzig": return 90;
|
||||
case "hundert": return 1;
|
||||
default: return 0
|
||||
}
|
||||
}
|
||||
const conv_str = (input: string) => {
|
||||
let result = parseInt(input);
|
||||
if (Number.isNaN(result)) {
|
||||
result = 0;
|
||||
} else {
|
||||
return result;
|
||||
}
|
||||
if (input.indexOf("hundert") >= 0) {
|
||||
let i1 = input.split("hundert")[0];
|
||||
input = input.split("hundert")[1];
|
||||
let i2 = txttonum_single(i1) * 100;
|
||||
result += i2 > 0 ? i2 : 100
|
||||
}
|
||||
if (input == "elf") {
|
||||
result += 11;
|
||||
} else if (input == "zwölf") {
|
||||
result += 12;
|
||||
} else if (input.indexOf("zehn") >= 0) {
|
||||
let i1 = txttonum_single(input.replace("zehn", ""))
|
||||
result += i1 + 10;
|
||||
} else if (input.indexOf("und") >= 0) {
|
||||
let i1 = input.split("und")[0];
|
||||
input = input.split("und")[1];
|
||||
result += txttonum_single(i1);
|
||||
result += txttonum_single(input);
|
||||
} else {
|
||||
result += txttonum_single(input);
|
||||
}
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
let verse_str = numeric_parts[1].split("bis").map(el => el.trim())
|
||||
if (verse_str.length == 1){
|
||||
res.json(`${book}/${conv_str(numeric_parts[0])}/${conv_str(verse_str[0])}`);
|
||||
}else{
|
||||
res.json(`${book}/${conv_str(numeric_parts[0])}/${conv_str(verse_str[0])}-${conv_str(verse_str[1])}`);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
124
api-gateway/src/controllers/translation.controller.ts
Normal file
124
api-gateway/src/controllers/translation.controller.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
import { BibleProvider } from '../providers/bible.provider.js';
|
||||
import { request } from 'node:http';
|
||||
|
||||
export namespace TranslationController {
|
||||
export const getBookJSON = (req: Request, res: Response, next: NextFunction) => {
|
||||
let data = BibleProvider.get_book(req.params.translation, req.params.book);
|
||||
|
||||
if (!data) {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "book",
|
||||
data: {
|
||||
message: `book ${req.params.book} not found in ${req.params.translation}`
|
||||
},
|
||||
success: false
|
||||
});
|
||||
} else {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "book",
|
||||
data: {
|
||||
translation: req.params.translation,
|
||||
book: req.params.book,
|
||||
chapters: data
|
||||
},
|
||||
success: true
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const getChapterJSON = (req: Request, res: Response, next: NextFunction) => {
|
||||
let data = BibleProvider.get_chapter(req.params.translation, req.params.book, parseInt(req.params.chapter));
|
||||
if (!data || data.length === 0) {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "chapter",
|
||||
data: {
|
||||
message: `chapter ${req.params.chapter} of book ${req.params.book} not found in ${req.params.translation}`
|
||||
},
|
||||
success: false
|
||||
});
|
||||
} else {
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "chapter",
|
||||
data: {
|
||||
translation: req.params.translation,
|
||||
book: req.params.book,
|
||||
chapter: req.params.chapter,
|
||||
verses: data
|
||||
},
|
||||
success: true
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export const getVerseJSON = (req: Request, res: Response, next: NextFunction) => {
|
||||
let data = ""
|
||||
let verses = [];
|
||||
if (req.params.verse.includes("-")) {
|
||||
// handle range
|
||||
let ranges = req.params.verse.split('-');
|
||||
for (let v = parseInt(ranges[0]); v <= parseInt(ranges[1]); v++) {
|
||||
let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), v);
|
||||
if (verse_obj) {
|
||||
verses.push(verse_obj)
|
||||
data += `${verse_obj.text} `;
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), parseInt(req.params.verse));
|
||||
if (verse_obj) {
|
||||
verses.push(verse_obj)
|
||||
data = verse_obj.text;
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "verse",
|
||||
data: {
|
||||
translation: req.params.translation,
|
||||
book: req.params.book,
|
||||
chapter: req.params.chapter,
|
||||
verse: req.params.verse,
|
||||
text: data.trimEnd(),
|
||||
verses: verses
|
||||
},
|
||||
success: true
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
export const getVerseRaw = (req: Request, res: Response, next: NextFunction) => {
|
||||
let data = ""
|
||||
if (req.params.verse.includes("-")) {
|
||||
// handle range
|
||||
let ranges = req.params.verse.split('-');
|
||||
for (let v = parseInt(ranges[0]); v <= parseInt(ranges[1]); v++) {
|
||||
let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), v);
|
||||
if (verse_obj) {
|
||||
data += `${verse_obj.text} `;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let verse_obj = BibleProvider.get_verse(req.params.translation, req.params.book, parseInt(req.params.chapter), parseInt(req.params.verse));
|
||||
if (verse_obj) {
|
||||
data = verse_obj.text;
|
||||
}
|
||||
}
|
||||
console.log("test")
|
||||
if (!data) {
|
||||
res.status(404).json(`Kapitel ${req.params.chapter} vers ${req.params.verse} vom buch ${req.params.book} konnte in der Übersetzung ${req.params.translation} nicht gefunden werden.`);
|
||||
} else {
|
||||
res.charset = "UTF-8"
|
||||
res.json(data);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
61
api-gateway/src/index.ts
Normal file
61
api-gateway/src/index.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import { BibleProvider } from "./providers/bible.provider.js";
|
||||
import express from 'express';
|
||||
import defaultRouter from "./routes/default.routes.js";
|
||||
import config from "./config/config.js";
|
||||
import helmet from "helmet";
|
||||
import { exit } from "process";
|
||||
import { DownloaderProvider } from "./providers/downloader.provider.js";
|
||||
|
||||
//BibleProvider.load_biliothek();
|
||||
//console.log(`Loaded ${BibleProvider.biliothek.length} translations in biliothek`);
|
||||
|
||||
|
||||
//const downloader = new DownloaderProvider.Downloader('SLT');
|
||||
//downloader.start();
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use(helmet())
|
||||
app.disable('x-powered-by')
|
||||
|
||||
app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||
// log accessed url
|
||||
// calculate processing time
|
||||
const start = Date.now();
|
||||
next();
|
||||
const duration = Date.now() - start;
|
||||
const access_log = {
|
||||
date: new Date().toISOString(),
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
ip: req.ip,
|
||||
agent: req.headers['user-agent'] || '',
|
||||
status: res.statusCode,
|
||||
duration: duration
|
||||
}
|
||||
console.log(access_log);
|
||||
|
||||
|
||||
});
|
||||
|
||||
app.use('/', defaultRouter);
|
||||
|
||||
|
||||
|
||||
app.use((err: any, req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||
console.error(err.stack);
|
||||
res.status(500).json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "error",
|
||||
data: {
|
||||
message: 'Internal Server Error'
|
||||
},
|
||||
success: false
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
app.listen(config.port, () => {
|
||||
console.log(`Server running on port ${config.port}`);
|
||||
});
|
||||
|
||||
183
api-gateway/src/providers/bible.provider.ts
Normal file
183
api-gateway/src/providers/bible.provider.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
import * as fs from 'fs';
|
||||
import NodeCache from 'node-cache';
|
||||
import { DownloaderProvider } from './downloader.provider.js';
|
||||
|
||||
export namespace BibleProvider {
|
||||
|
||||
let cache = new NodeCache();
|
||||
|
||||
export interface Translation {
|
||||
translation: string;
|
||||
books: Book[];
|
||||
}
|
||||
|
||||
export interface Book {
|
||||
book: string;
|
||||
chapters: Chapter[];
|
||||
}
|
||||
|
||||
export interface Chapter {
|
||||
translation?: string;
|
||||
book?: string;
|
||||
chapter?: number;
|
||||
verse: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
setInterval(() => {
|
||||
let completed_downloads = downloads.filter(el => el.getStatus() == "completed" || el.getStatus() == "error")
|
||||
for (let completed_download of completed_downloads) {
|
||||
if (completed_download.time_end) {
|
||||
const time = new Date();
|
||||
const time_since = (time.getTime() - completed_download.time_end.getTime()) / 1000;
|
||||
if (time_since > 60 * 15) {
|
||||
const index = downloads.indexOf(completed_download)
|
||||
if (index > -1) { // only splice array when item is found
|
||||
downloads.splice(index, 1);
|
||||
}
|
||||
console.log("Removed old download item")
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 60000) // called every minute
|
||||
|
||||
let downloads: DownloaderProvider.Downloader[] = []
|
||||
|
||||
export const get_all_book_names = () => {
|
||||
let books: string[] | undefined = cache.get(`books`);
|
||||
if (books == undefined) {
|
||||
const book_list_file = `data/books.json`;
|
||||
books = JSON.parse(fs.readFileSync(book_list_file, 'utf-8')).books;
|
||||
cache.set('books', books, 60000)
|
||||
}
|
||||
return books;
|
||||
}
|
||||
|
||||
export const start_download = (translation: string) => {
|
||||
let existing_download = downloads.filter((el: DownloaderProvider.Downloader) => el.translation == translation);
|
||||
if (existing_download.length > 0) {
|
||||
return {
|
||||
status: existing_download[0].getStatus(),
|
||||
uuid: existing_download[0].uuid,
|
||||
logs: existing_download[0].logs
|
||||
}
|
||||
} else {
|
||||
let downloader = new DownloaderProvider.Downloader(translation)
|
||||
downloads.push(downloader);
|
||||
downloader.start();
|
||||
return {
|
||||
status: downloader.getStatus(),
|
||||
uuid: downloader.uuid,
|
||||
logs: downloader.logs
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export const list_translations = (): string[] => {
|
||||
let translations: string[] | undefined = cache.get('translations')
|
||||
if (translations == undefined) {
|
||||
console.log("Translations not in cache. Load from disk")
|
||||
// load translations from disk
|
||||
translations = []
|
||||
let folders = fs.readdirSync('data', { withFileTypes: true })
|
||||
for (let folder of folders) {
|
||||
if (folder.isDirectory()) {
|
||||
translations.push(folder.name)
|
||||
}
|
||||
}
|
||||
cache.set('translations', translations, 300)
|
||||
}
|
||||
return translations;
|
||||
}
|
||||
|
||||
export const get_translation = (translation: string): Translation | undefined => {
|
||||
let translation_obj: Translation | undefined = cache.get(`translation:${translation}`);
|
||||
if (translation_obj == undefined) {
|
||||
console.log("Translation not in cache. Load from disk")
|
||||
|
||||
translation_obj = {
|
||||
translation: translation,
|
||||
books: []
|
||||
}
|
||||
// load book names
|
||||
let files = fs.readdirSync(`data/${translation}`, { withFileTypes: true });
|
||||
for (let file of files) {
|
||||
if (file.isFile() && file.name.endsWith('.json')) {
|
||||
let book_name = file.name.replace('.json', '');
|
||||
translation_obj.books.push({
|
||||
book: book_name,
|
||||
chapters: []
|
||||
})
|
||||
}
|
||||
}
|
||||
cache.set(`translation:${translation}`, translation_obj, 300)
|
||||
}
|
||||
return translation_obj;
|
||||
}
|
||||
|
||||
export const get_books = (translation: string): string[] | undefined => {
|
||||
let t = get_translation(translation);
|
||||
if (!t) return undefined;
|
||||
return t.books.map(b => b.book);
|
||||
}
|
||||
|
||||
export const get_book = (translation: string, book: string) => {
|
||||
let t = get_translation(translation);
|
||||
if (!t) return undefined;
|
||||
let book_obj: Book | undefined = cache.get(`translation:${translation}:${book}`)
|
||||
if (book_obj == undefined) {
|
||||
book_obj = {
|
||||
book: book,
|
||||
chapters: []
|
||||
}
|
||||
console.log("Book not in cache. Load from disk")
|
||||
try {
|
||||
let book_file = `${book}.json`
|
||||
let book_content = JSON.parse(fs.readFileSync(`data/${translation}/${book_file}`, 'utf-8'));
|
||||
if (!book_content || book_content.length === 0) {
|
||||
cache.set(`translation:${translation}:${book}`, null, 300) // set nul to mark that it does not exist
|
||||
return undefined;
|
||||
}
|
||||
book_obj.chapters = book_content.map((c: any) => {
|
||||
return { chapter: c.chapter, verse: c.verse, text: c.text };
|
||||
});
|
||||
cache.set(`translation:${translation}:${book}`, book_obj, 300)
|
||||
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
return book_obj?.chapters;
|
||||
}
|
||||
|
||||
export const get_chapter = (translation: string, book: string, chapter: number) => {
|
||||
let chapter_obj: Chapter[] | undefined = cache.get(`translation:${translation}:${book}:${chapter}`)
|
||||
if (chapter_obj == undefined) {
|
||||
console.log("Chapter not in cache. Generate data.")
|
||||
let b = get_book(translation, book);
|
||||
if (!b) {
|
||||
cache.set(`translation:${translation}:${book}:${chapter}`, null, 300);
|
||||
return undefined
|
||||
}
|
||||
chapter_obj = b.filter(c => c.chapter === chapter).map(c => {
|
||||
return { verse: c.verse, text: c.text };
|
||||
});
|
||||
cache.set(`translation:${translation}:${book}:${chapter}`, chapter_obj, 300);
|
||||
|
||||
}
|
||||
return chapter_obj;
|
||||
}
|
||||
|
||||
|
||||
export const get_verse = (translation: string, book: string, chapter: number, verse: number): Chapter | undefined => {
|
||||
let chapters = get_chapter(translation, book, chapter);
|
||||
if (!chapters) return undefined;
|
||||
return chapters.find(v => v.verse === verse);
|
||||
}
|
||||
|
||||
export const get_cache_metrics = () => {
|
||||
return cache.getStats();
|
||||
}
|
||||
}
|
||||
185
api-gateway/src/providers/downloader.provider.ts
Normal file
185
api-gateway/src/providers/downloader.provider.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
import axios from 'axios';
|
||||
import { randomUUID } from 'crypto';
|
||||
import * as fs from 'fs';
|
||||
import { parse } from 'node-html-parser';
|
||||
import config from '../config/config.js';
|
||||
|
||||
export namespace DownloaderProvider {
|
||||
const bibleserver_endpoint = 'https://www.bibleserver.com';
|
||||
|
||||
export class Downloader {
|
||||
private _translation: string;
|
||||
private _logs: string[] = [];
|
||||
private _status: 'idle' | 'running' | 'completed' | 'error' = 'idle';
|
||||
private _books: string[] = [];
|
||||
private _data_directory: string = 'data';
|
||||
private _operation_id: string = ''
|
||||
private _time_start: Date | undefined;
|
||||
private _time_end: Date | undefined;
|
||||
|
||||
public get translation() {
|
||||
return this._translation
|
||||
}
|
||||
|
||||
public get uuid() {
|
||||
return this._operation_id
|
||||
}
|
||||
|
||||
public get logs() {
|
||||
return this._logs
|
||||
}
|
||||
|
||||
public get time_start() {
|
||||
return this._time_start
|
||||
}
|
||||
|
||||
public get time_end() {
|
||||
return this._time_end
|
||||
}
|
||||
|
||||
constructor(private translation_in: string) {
|
||||
this._translation = translation_in;
|
||||
this._operation_id = randomUUID()
|
||||
this._status = 'idle';
|
||||
this.log(`Initialized downloader for translation: ${this._translation}`);
|
||||
this.log(`Using bibleserver endpoint: ${bibleserver_endpoint}`);
|
||||
|
||||
}
|
||||
|
||||
public start() {
|
||||
this._time_start = new Date();
|
||||
this.log(`Starting download for translation: ${this._translation}`);
|
||||
this._status = 'running';
|
||||
// get all translations from reference file
|
||||
try {
|
||||
const book_list_file = `${this._data_directory}/books.json`;
|
||||
this.log(`Loading book list from ${book_list_file}`);
|
||||
this._books = JSON.parse(fs.readFileSync(book_list_file, 'utf-8')).books;
|
||||
this.log(`Loaded ${this._books.length} books to download`);
|
||||
} catch (error) {
|
||||
this.log(`Error loading book list: ${error}`);
|
||||
this._status = 'error';
|
||||
return;
|
||||
}
|
||||
// create directory
|
||||
try {
|
||||
if (!fs.existsSync(`${this._data_directory}/${this._translation}`)) {
|
||||
fs.mkdirSync(`${this._data_directory}/${this._translation}`, { recursive: true });
|
||||
this.log(`Created directory: ${this._data_directory}/${this._translation}`);
|
||||
}
|
||||
} catch (error) {
|
||||
this.log(`Error creating translation directory: ${error}`);
|
||||
this._status = 'error';
|
||||
return;
|
||||
}
|
||||
|
||||
this.fetch_all_books();
|
||||
|
||||
}
|
||||
|
||||
public getStatus(): 'idle' | 'running' | 'completed' | 'error' {
|
||||
return this._status;
|
||||
}
|
||||
|
||||
private delay(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
private async fetch_all_books() {
|
||||
try {
|
||||
for (let book of this._books) {
|
||||
if (fs.existsSync(`${this._data_directory}/${this._translation}/${book}.json`)) {
|
||||
this.log(`Book ${book} already exists for ${this._translation}, skipping`);
|
||||
continue;
|
||||
}
|
||||
this.log(`Fetching book: ${book} for translation: ${this._translation}`);
|
||||
|
||||
let chapter = 1;
|
||||
let book_content: any[] = [];
|
||||
|
||||
while (true) {
|
||||
this.log(`Trying to fetch chapter ${chapter} of book ${book}`);
|
||||
await this.delay(config.downloadDelay * 1000); // sleep to prevent DDOS
|
||||
let chapter_content: any = await this.fetch_chapter(book, chapter);
|
||||
if (chapter_content.length === 0) {
|
||||
break;
|
||||
}
|
||||
book_content = book_content.concat(chapter_content);
|
||||
chapter++;
|
||||
}
|
||||
fs.writeFileSync(`${this._data_directory}/${this._translation}/${book}.json`, JSON.stringify(book_content, null, 4));
|
||||
this.log(`Saved book ${book} for translation ${this._translation} with ${book_content.length} verses`);
|
||||
this.log(`Completed fetching book: ${book} for translation: ${this._translation}`);
|
||||
}
|
||||
this._time_end = new Date();
|
||||
this._status = "completed"
|
||||
} catch (error) {
|
||||
this.log(`Error fetching books: ${error}`);
|
||||
this._status = 'error';
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private async fetch_chapter(book: string, chapter: number) {
|
||||
let bibleserver_url = `${bibleserver_endpoint}/${this._translation}/${book}${chapter}`;
|
||||
this.log(`Fetching URL from: ${bibleserver_url}`)
|
||||
try {
|
||||
let response = await axios.get(bibleserver_url)
|
||||
this.log("Received response")
|
||||
let html = response.data;
|
||||
let root = parse(html);
|
||||
// verify to avoid redirect
|
||||
let book_verify_name = root.querySelector('.chapter')?.querySelector('header')?.querySelector('h1')?.text.trim() || '';
|
||||
if (book_verify_name !== `${book} ${chapter}`) {
|
||||
// chapter does not exist, return empty list
|
||||
return [];
|
||||
}
|
||||
let verse_elements = root.querySelectorAll('.verse');
|
||||
let result_array = [];
|
||||
for (let verse_element of verse_elements) {
|
||||
verse_element.querySelectorAll('.footnote').forEach(fn => fn.remove()); // remove footnotes
|
||||
let verse_raw = verse_element.querySelector('.verse-number')?.childNodes[0].text
|
||||
// resolve verse ranges
|
||||
if (verse_raw?.includes('-')) {
|
||||
let ranges = verse_raw.split('-');
|
||||
for (let v = parseInt(ranges[0]); v <= parseInt(ranges[1]); v++) {
|
||||
result_array.push({
|
||||
translation: this._translation,
|
||||
book: book,
|
||||
chapter: chapter,
|
||||
verse: v,
|
||||
text: verse_element.querySelector('.verse-content')?.childNodes[0].text || ''
|
||||
});
|
||||
}
|
||||
} else {
|
||||
result_array.push({
|
||||
translation: this._translation,
|
||||
book: book,
|
||||
chapter: chapter,
|
||||
verse: Number(verse_element.querySelector('.verse-number')?.childNodes[0].text) || -1,
|
||||
text: verse_element.querySelector('.verse-content')?.childNodes[0].text || ''
|
||||
});
|
||||
}
|
||||
}
|
||||
return result_array
|
||||
} catch (error: any) {
|
||||
if (error.response && error.response.status === 404) {
|
||||
// translation does not exist
|
||||
this.log(`Translation ${this._translation} does not exist for book ${book}`);
|
||||
return [];
|
||||
} else {
|
||||
this.log(`Error fetching ${this._translation} ${book} ${chapter}- ${error}`);
|
||||
this.log(`URL: ${bibleserver_url}`);
|
||||
throw (error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private log(message: string) {
|
||||
let log_entry = `[Downloader<${this._operation_id}>:${this._translation}][${new Date().toISOString()}] ${message}`;
|
||||
this._logs.push(log_entry);
|
||||
console.log(log_entry);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
33
api-gateway/src/routes/default.routes.ts
Normal file
33
api-gateway/src/routes/default.routes.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { NextFunction, Router, Request, Response } from "express";
|
||||
import { TranslationController } from "../controllers/translation.controller.js";
|
||||
import { LibraryController } from "../controllers/book.controller.js";
|
||||
import { GenericController } from "../controllers/generic.controller.js";
|
||||
import { SpeechController } from "../controllers/speech.controller.js";
|
||||
|
||||
const defaultRouter = Router()
|
||||
|
||||
defaultRouter.get('/_statsCache',GenericController.cacheStats);
|
||||
defaultRouter.get('/_convert/:inputstring',SpeechController.translateText);
|
||||
|
||||
|
||||
defaultRouter.get('/', LibraryController.listTranslationsJSON);
|
||||
defaultRouter.get('/:translation', LibraryController.getTranslationJSON);
|
||||
defaultRouter.get('/:translation/_pull', LibraryController.pullTranslation);
|
||||
defaultRouter.get('/:translation/:book', TranslationController.getBookJSON);
|
||||
defaultRouter.get('/:translation/:book/:chapter', TranslationController.getChapterJSON);
|
||||
defaultRouter.get('/:translation/:book/:chapter/:verse', TranslationController.getVerseJSON);
|
||||
defaultRouter.get('/:translation/:book/:chapter/:verse/raw', TranslationController.getVerseRaw);
|
||||
|
||||
|
||||
defaultRouter.use((err: any, req: Request, res: Response, next: NextFunction) => {
|
||||
console.log(err)
|
||||
res.status(404).json({
|
||||
requested_at: new Date().toISOString(),
|
||||
rescource: "error",
|
||||
data: {
|
||||
message: 'Resource not found'
|
||||
},
|
||||
success: false
|
||||
});
|
||||
});
|
||||
export default defaultRouter;
|
||||
38
api-gateway/tsconfig.json
Normal file
38
api-gateway/tsconfig.json
Normal file
@@ -0,0 +1,38 @@
|
||||
{
|
||||
// Visit https://aka.ms/tsconfig to read more about this file
|
||||
"compilerOptions": {
|
||||
// File Layout
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
// Environment Settings
|
||||
// See also https://aka.ms/tsconfig/module
|
||||
"module": "nodenext",
|
||||
"target": "esnext",
|
||||
"moduleResolution": "nodenext",
|
||||
|
||||
// For nodejs:
|
||||
// "lib": ["esnext"],
|
||||
// "types": ["node"],
|
||||
// and npm install -D @types/node
|
||||
// Other Outputs
|
||||
"sourceMap": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
// Stricter Typechecking Options
|
||||
"exactOptionalPropertyTypes": true,
|
||||
// Style Options
|
||||
// "noImplicitReturns": true,
|
||||
// "noImplicitOverride": true,
|
||||
// "noUnusedLocals": true,
|
||||
// "noUnusedParameters": true,
|
||||
// "noFallthroughCasesInSwitch": true,
|
||||
// "noPropertyAccessFromIndexSignature": true,
|
||||
// Recommended Options
|
||||
"strict": true,
|
||||
"jsx": "react-jsx",
|
||||
"verbatimModuleSyntax": false,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"skipLibCheck": true,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user