From ad4028e35a5bb49bd04149be66af7166c05054fe Mon Sep 17 00:00:00 2001 From: Amaury Joly Date: Sun, 25 Apr 2021 22:48:57 +0200 Subject: [PATCH] adding listener rss package + better addNewListener's implementation + better sqlite management --- package-lock.json | 104 +++++++++++++++++------- package.json | 11 ++- src/manage-listener.ts | 20 ++--- src/sqlite-tools.ts | 87 ++++++++++---------- tests/index-spec.ts | 177 +---------------------------------------- 5 files changed, 146 insertions(+), 253 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4597cae..30f1feb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "license": "MIT", "dependencies": { "@databases/sqlite": "^3.0.0", - "listener-rss": "file:../listener-rss" + "listener-rss": "^0.0.1" }, "devDependencies": { "@types/chai": "^4.2.15", @@ -37,6 +37,7 @@ }, "../listener-rss": { "version": "1.0.0", + "extraneous": true, "license": "MIT", "dependencies": { "rss-parser": "3.11.0" @@ -1129,6 +1130,14 @@ "node": ">=8.6" } }, + "node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -2306,8 +2315,12 @@ } }, "node_modules/listener-rss": { - "resolved": "../listener-rss", - "link": true + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/listener-rss/-/listener-rss-0.0.1.tgz", + "integrity": "sha512-ljH6FD4NMypkrZ9ZIMdW2YdAlm/Jer++zPI+1HTWK7BbhPJmUF3sSymqUqAjBcfH3xjt+fpYPqYgNYNM1fEYpA==", + "dependencies": { + "rss-parser": "3.11.0" + } }, "node_modules/load-json-file": { "version": "2.0.0", @@ -3443,6 +3456,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rss-parser": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.11.0.tgz", + "integrity": "sha512-oTLoYW+bNqNwkz8OpGinBU9s3As0sdczQjETIZFgyAdi7AopyhoVFGPIyFMYXXEY8hayKzD5CH+4CtmiPtJ89g==", + "dependencies": { + "entities": "^2.0.3", + "xml2js": "^0.4.19" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -4239,6 +4261,26 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "node_modules/xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -5149,6 +5191,11 @@ "ansi-colors": "^4.1.1" } }, + "entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==" + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -6042,31 +6089,11 @@ } }, "listener-rss": { - "version": "file:../listener-rss", + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/listener-rss/-/listener-rss-0.0.1.tgz", + "integrity": "sha512-ljH6FD4NMypkrZ9ZIMdW2YdAlm/Jer++zPI+1HTWK7BbhPJmUF3sSymqUqAjBcfH3xjt+fpYPqYgNYNM1fEYpA==", "requires": { - "@types/chai": "^4.2.14", - "@types/mocha": "^8.2.0", - "@types/node": "^14.14.25", - "@typescript-eslint/eslint-plugin": "^4.14.2", - "@typescript-eslint/parser": "^4.14.2", - "chai": "4.3.0", - "cross-env": "7.0.3", - "eslint": "^7.19.0", - "eslint-config-airbnb-base": "^14.2.1", - "eslint-config-prettier": "7.2.0", - "eslint-plugin-import": "^2.22.1", - "eslint-plugin-mocha": "8.0.0", - "eslint-plugin-prettier": "3.3.1", - "mocha": "8.2.1", - "prettier": "2.2.1", - "proxyquire": "2.1.3", - "rss-parser": "3.11.0", - "sinon-chai": "3.5.0", - "ts-mock-imports": "1.3.3", - "ts-node": "9.1.1", - "ts-sinon": "2.0.1", - "tsc-watch": "^4.2.9", - "typescript": "^4.1.3" + "rss-parser": "3.11.0" } }, "load-json-file": { @@ -6947,6 +6974,15 @@ "glob": "^7.1.3" } }, + "rss-parser": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.11.0.tgz", + "integrity": "sha512-oTLoYW+bNqNwkz8OpGinBU9s3As0sdczQjETIZFgyAdi7AopyhoVFGPIyFMYXXEY8hayKzD5CH+4CtmiPtJ89g==", + "requires": { + "entities": "^2.0.3", + "xml2js": "^0.4.19" + } + }, "run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -7568,6 +7604,20 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 4bdada0..c15ab30 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,18 @@ "description": "", "main": "build/index.js", "types": "build/index.d.ts", + "files": [ + "build/" + ], "scripts": { "test": "cross-env TS_NODE_PROJECT='./tests/tsconfig.json' mocha --require ts-node/register ./tests/**/*-spec.ts", "build": "tsc -p ./src" }, - "keywords": [], + "keywords": [ + "RSS", + "RSS to JSON", + "RSS listener" + ], "author": "Amaury Joly ", "license": "MIT", "devDependencies": { @@ -36,6 +43,6 @@ }, "dependencies": { "@databases/sqlite": "^3.0.0", - "listener-rss": "file:../listener-rss" + "listener-rss": "^0.0.1" } } diff --git a/src/manage-listener.ts b/src/manage-listener.ts index fcafe50..4bbc1c4 100644 --- a/src/manage-listener.ts +++ b/src/manage-listener.ts @@ -22,24 +22,24 @@ export class ManageListener extends EventEmitter { const configs: ListenerRSSInfos[] = await this.sqliteDb.fetchAll(); configs.forEach((config) => { - const newListener = new ListenerRss(config); - this.listenerArray.push(newListener); - this.settingEvents(newListener); + this.addNewListener(config); }); } - settingEvents(newListener: ListenerRss): void { + private addNewListener(info: ListenerRSSInfos) : ListenerRss { + const newListener = new ListenerRss(info); + this.listenerArray.push(newListener); + newListener.on("update", (obj) => this.emit("update", obj)); newListener.on("newEntries", (obj) => this.emit("newEntries", obj)); newListener.on("error", (err) => this.emit("error", err)); + + return newListener; } - async addNewListener(info: ListenerRSSInfos) { - const newListener = new ListenerRss(info); - this.listenerArray.push(newListener); - this.settingEvents(newListener); - - await this.sqliteDb.insertListener(newListener); + async registerListener(info: ListenerRSSInfos) { + const listener = this.addNewListener(info); + await this.sqliteDb.insertListener(listener); } async save() { diff --git a/src/sqlite-tools.ts b/src/sqlite-tools.ts index e84ed7c..a5c8ae9 100644 --- a/src/sqlite-tools.ts +++ b/src/sqlite-tools.ts @@ -1,6 +1,6 @@ import { ListenerRSSInfos } from "listener-rss"; -import connect, { sql } from "@databases/sqlite"; +import connect, { DatabaseConnection, sql } from "@databases/sqlite"; export class SqliteTools { path?: string; @@ -9,30 +9,39 @@ export class SqliteTools { this.path = path; } - async ensureTableExists() { + async withDB(callback: (db: DatabaseConnection) => Promise): Promise { const db = connect(this.path); + try { + return callback(db); + } finally { + await db.dispose(); + } + } - let req = sql`CREATE TABLE IF NOT EXISTS listeners - ( - address TEXT NOT NULL UNIQUE, - customfields TEXT DEFAULT '[]' NOT NULL, - timeloop INTEGER DEFAULT 300 NOT NULL, - last_entries_links TEXT DEFAULT '[]' NOT NULL, - PRIMARY KEY (address), - CHECK(timeloop >= 0) - );`; - - await db.query(req); - await db.dispose(); + async ensureTableExists() { + await this.withDB(async (db) => { + let req = sql`CREATE TABLE IF NOT EXISTS listeners + ( + address TEXT NOT NULL, + customfields TEXT DEFAULT '[]' NOT NULL, + timeloop INTEGER DEFAULT 300 NOT NULL, + last_entries_links TEXT DEFAULT '[]' NOT NULL, + PRIMARY KEY (address), + CHECK(timeloop >= 0) + );`; + + await db.query(req); + }); } async fetchAll(): Promise { - const db = connect(this.path); + + const rows = await this.withDB(async (db) => { + let req = sql`SELECT * + FROM listeners`; + return await db.query(req); + }); - let req = sql`SELECT * - FROM listeners`; - const rows = await db.query(req); - await db.dispose(); return rows.map((row: any) => ({ address: row["address"], customfields: JSON.parse(row["customfields"]), @@ -42,30 +51,28 @@ export class SqliteTools { } async insertListener(listener: ListenerRSSInfos) { - const db = connect(this.path); - - let req = sql`INSERT INTO listeners (address, timeloop, customfields, last_entries_links) - VALUES (${listener.address}, - ${listener.timeloop}, - ${JSON.stringify(listener.customfields ? listener.customfields : [])}, - ${JSON.stringify(listener.lastEntriesLinks ? listener.lastEntriesLinks : [])})`; - await db.query(req); - await db.dispose(); + await this.withDB(async (db) => { + let req = sql`INSERT INTO listeners (address, timeloop, customfields, last_entries_links) + VALUES (${listener.address}, + ${listener.timeloop}, + ${JSON.stringify(listener.customfields ?? [])}, + ${JSON.stringify(listener.lastEntriesLinks ?? [])})`; + await db.query(req); + }); } async updateAll(listeners: ListenerRSSInfos[]) { - const db = connect(this.path); - - await db.tx(async (transaction) => { - for (const listener of listeners) { - let req = sql`UPDATE listeners - SET last_entries_links = ${JSON.stringify( - listener.lastEntriesLinks - )} - WHERE address = ${listener.address}`; - await transaction.query(req); - } + await this.withDB(async (db) => { + await db.tx(async (transaction) => { + for (const listener of listeners) { + let req = sql`UPDATE listeners + SET last_entries_links = ${JSON.stringify( + listener.lastEntriesLinks + )} + WHERE address = ${listener.address}`; + await transaction.query(req); + } + }); }); - await db.dispose(); } } diff --git a/tests/index-spec.ts b/tests/index-spec.ts index 00eb220..99f9bb4 100644 --- a/tests/index-spec.ts +++ b/tests/index-spec.ts @@ -32,7 +32,7 @@ describe("test class ManageListener", function () { const ml = new ManageListener(path); await ml.load(); - for (const item of dataWithoutHistory) await ml.addNewListener(item); + for (const item of dataWithoutHistory) await ml.registerListener(item); // when ml.save(); @@ -62,7 +62,7 @@ describe("test class ManageListener", function () { const clock = sinon.useFakeTimers(); const ml = new ManageListener(path); await ml.load(); - for (const item of dataWithoutHistory) await ml.addNewListener(item); + for (const item of dataWithoutHistory) await ml.registerListener(item); const updateSpy = sinon.spy(); const newEntriesSpy = sinon.spy(); @@ -152,7 +152,7 @@ describe("test class ManageListener", function () { const clock = sinon.useFakeTimers(); const ml = new ManageListener(path); await ml.load(); - for (const item of dataWithHistory) await ml.addNewListener(item); + for (const item of dataWithHistory) await ml.registerListener(item); const updateSpy = sinon.spy(); const newEntriesSpy = sinon.spy(); @@ -251,176 +251,5 @@ describe("test class ManageListener", function () { ml.stopAll(); }); }); - - // it("should call correctly the events (with aggregation)", async function () { - // // given - // const clock = sinon.useFakeTimers(); - // const ml = new ManageListener({ - // timeloop: 30, - // path: "tests/RessourcesTest/RealRessources/save_no_history.json", - // }); - - // const updateSpy = sinon.spy((obj) => console.log(obj)); - // const newEntriesSpy = sinon.spy(); - - // const tabChannelId = [ - // "UCOuIgj0CYCXCvjWywjDbauw", - // "UCh2YBKhYIy-_LtfCIn2Jycg", - // "UCY7klexd1qEqxgqYK6W7BVQ", - // ]; - - // ml.on("update", updateSpy); - // ml.on("newEntries", newEntriesSpy); - // tabChannelId.forEach((item: string) => { - // nock("https://www.youtube.com") - // .get(`/feeds/videos.xml?channel_id=${item}`) - // .replyWithFile( - // 200, - // path.join(__dirname, `RessourcesTest/RealRessources/${item}.rss`), - // { "content-type": "text/xml", charset: "utf-8" } - // ) - // .persist(); - // }); - // ml.startAll(); - - // // when - // await Promise.all([ - // events.once(ml.listenerArray[0], "update"), - // events.once(ml.listenerArray[1], "update"), - // events.once(ml.listenerArray[2], "update"), - // ]); - - // //expect - // expect(updateSpy).to.have.been.calledOnce; - // expect(newEntriesSpy).to.have.been.calledOnce; - - // // given - // updateSpy.resetHistory(); - // newEntriesSpy.resetHistory(); - - // // when - // await clock.tickAsync(10000); - // await events.once(ml.listenerArray[0], "update"); - - // //expect - // expect(updateSpy).to.not.have.been.called; - // expect(newEntriesSpy).to.not.have.been.called; - - // // when - // await clock.tickAsync(10000); - // await Promise.all([ - // events.once(ml.listenerArray[0], "update"), - // events.once(ml.listenerArray[1], "update"), - // ]); - - // //expect - // expect(updateSpy).to.not.have.been.called; - // expect(newEntriesSpy).to.not.have.been.called; - - // // when - // await clock.tickAsync(10000); - // await Promise.all([ - // events.once(ml.listenerArray[0], "update"), - // events.once(ml.listenerArray[2], "update"), - // ]); - - // //expect - // expect(updateSpy).to.have.been.calledOnce; - // expect(newEntriesSpy).to.not.have.been.called; - - // ml.stopAll(); - // }); - - // it("should call correctly the events with a new entry (with aggregation)", async function () { - // // given - // const clock = sinon.useFakeTimers(); - // const ml = new ManageListener({ - // timeloop: 30, - // path: "tests/RessourcesTest/RealRessources/save.json", - // }); - - // const updateSpy = sinon.spy(); - // const newEntriesSpy = sinon.spy(); - - // const tabChannelId = [ - // "UCOuIgj0CYCXCvjWywjDbauw", - // "UCh2YBKhYIy-_LtfCIn2Jycg", - // "UCY7klexd1qEqxgqYK6W7BVQ", - // ]; - - // ml.on("update", updateSpy); - // ml.on("newEntries", newEntriesSpy); - // tabChannelId.forEach((item: string) => { - // nock("https://www.youtube.com") - // .get(`/feeds/videos.xml?channel_id=${item}`) - // .once() - // .replyWithFile( - // 200, - // path.join(__dirname, `RessourcesTest/RealRessources/${item}.rss`), - // { "content-type": "text/xml", charset: "utf-8" } - // ); - // }); - // ml.startAll(); - - // // when - // await Promise.all([ - // events.once(ml.listenerArray[0], "update"), - // events.once(ml.listenerArray[1], "update"), - // events.once(ml.listenerArray[2], "update"), - // ]); - - // //expect - // expect(updateSpy).to.have.been.calledOnce; - // expect(newEntriesSpy).to.not.have.been.called; - - // // given - // updateSpy.resetHistory(); - - // tabChannelId.forEach((item: string) => { - // nock("https://www.youtube.com") - // .get(`/feeds/videos.xml?channel_id=${item}`) - // .replyWithFile( - // 200, - // path.join( - // __dirname, - // `RessourcesTest/RealRessources/WithUpdate/${item}.rss` - // ), - // { "content-type": "text/xml", charset: "utf-8" } - // ) - // .persist(); - // }); - - // // when - // await clock.tickAsync(10000); - // await events.once(ml.listenerArray[0], "update"); - - // //expect - // expect(updateSpy).to.not.have.been.called; - // expect(newEntriesSpy).to.not.have.been.called; - - // // when - // await clock.tickAsync(10000); - // await Promise.all([ - // events.once(ml.listenerArray[0], "update"), - // events.once(ml.listenerArray[1], "update"), - // ]); - - // //expect - // expect(updateSpy).to.not.have.been.called; - // expect(newEntriesSpy).to.not.have.been.called; - - // // when - // await clock.tickAsync(10000); - // await Promise.all([ - // events.once(ml.listenerArray[0], "update"), - // events.once(ml.listenerArray[2], "update"), - // ]); - - // //expect - // expect(updateSpy).to.have.been.calledOnce; - // expect(newEntriesSpy).to.have.been.calledOnce; - - // ml.stopAll(); - // }); }); });