amaury.joly/listener-rss-agregators-tests
1
0
Fork 0
forked from Outils-PeerTube/listener-rss-aggregator
Code Issues Pull Requests Projects Releases Wiki Activity

adding listener rss package + better addNewListener's implementation + better sqlite management

Browse Source
This commit is contained in:
Amaury Joly
2021-04-25 22:48:57 +02:00
parent 95a10dfc60
commit ad4028e35a
5 changed files with 146 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
package-lock.json generated
View File

@@ -9,7 +9,7 @@
"license": "MIT",
"dependencies": {
"@databases/sqlite": "^3.0.0",
"listener-rss": "file:../listener-rss"
"listener-rss": "^0.0.1"
},
"devDependencies": {
"@types/chai": "^4.2.15",
@@ -37,6 +37,7 @@
},
"../listener-rss": {
"version": "1.0.0",
"extraneous": true,
"license": "MIT",
"dependencies": {
"rss-parser": "3.11.0"
@@ -1129,6 +1130,14 @@
"node": ">=8.6"
}
},
"node_modules/entities": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
"integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==",
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/error-ex": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
@@ -2306,8 +2315,12 @@
}
},
"node_modules/listener-rss": {
"resolved": "../listener-rss",
"link": true
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/listener-rss/-/listener-rss-0.0.1.tgz",
"integrity": "sha512-ljH6FD4NMypkrZ9ZIMdW2YdAlm/Jer++zPI+1HTWK7BbhPJmUF3sSymqUqAjBcfH3xjt+fpYPqYgNYNM1fEYpA==",
"dependencies": {
"rss-parser": "3.11.0"
}
},
"node_modules/load-json-file": {
"version": "2.0.0",
@@ -3443,6 +3456,15 @@
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/rss-parser": {
"version": "3.11.0",
"resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.11.0.tgz",
"integrity": "sha512-oTLoYW+bNqNwkz8OpGinBU9s3As0sdczQjETIZFgyAdi7AopyhoVFGPIyFMYXXEY8hayKzD5CH+4CtmiPtJ89g==",
"dependencies": {
"entities": "^2.0.3",
"xml2js": "^0.4.19"
}
},
"node_modules/run-parallel": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@@ -4239,6 +4261,26 @@
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"node_modules/xml2js": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
"integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
"dependencies": {
"sax": ">=0.6.0",
"xmlbuilder": "~11.0.0"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
"engines": {
"node": ">=4.0"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
@@ -5149,6 +5191,11 @@
"ansi-colors": "^4.1.1"
}
},
"entities": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
"integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
},
"error-ex": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
@@ -6042,31 +6089,11 @@
}
},
"listener-rss": {
"version": "file:../listener-rss",
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/listener-rss/-/listener-rss-0.0.1.tgz",
"integrity": "sha512-ljH6FD4NMypkrZ9ZIMdW2YdAlm/Jer++zPI+1HTWK7BbhPJmUF3sSymqUqAjBcfH3xjt+fpYPqYgNYNM1fEYpA==",
"requires": {
"@types/chai": "^4.2.14",
"@types/mocha": "^8.2.0",
"@types/node": "^14.14.25",
"@typescript-eslint/eslint-plugin": "^4.14.2",
"@typescript-eslint/parser": "^4.14.2",
"chai": "4.3.0",
"cross-env": "7.0.3",
"eslint": "^7.19.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-config-prettier": "7.2.0",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-mocha": "8.0.0",
"eslint-plugin-prettier": "3.3.1",
"mocha": "8.2.1",
"prettier": "2.2.1",
"proxyquire": "2.1.3",
"rss-parser": "3.11.0",
"sinon-chai": "3.5.0",
"ts-mock-imports": "1.3.3",
"ts-node": "9.1.1",
"ts-sinon": "2.0.1",
"tsc-watch": "^4.2.9",
"typescript": "^4.1.3"
"rss-parser": "3.11.0"
}
},
"load-json-file": {
@@ -6947,6 +6974,15 @@
"glob": "^7.1.3"
}
},
"rss-parser": {
"version": "3.11.0",
"resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.11.0.tgz",
"integrity": "sha512-oTLoYW+bNqNwkz8OpGinBU9s3As0sdczQjETIZFgyAdi7AopyhoVFGPIyFMYXXEY8hayKzD5CH+4CtmiPtJ89g==",
"requires": {
"entities": "^2.0.3",
"xml2js": "^0.4.19"
}
},
"run-parallel": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@@ -7568,6 +7604,20 @@
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"xml2js": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
"integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
"requires": {
"sax": ">=0.6.0",
"xmlbuilder": "~11.0.0"
}
},
"xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
},
"y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

11
package.json
View File

@@ -4,11 +4,18 @@
"description": "",
"main": "build/index.js",
"types": "build/index.d.ts",
"files": [
"build/"
],
"scripts": {
"test": "cross-env TS_NODE_PROJECT='./tests/tsconfig.json' mocha --require ts-node/register ./tests/**/*-spec.ts",
"build": "tsc -p ./src"
},
"keywords": [],
"keywords": [
"RSS",
"RSS to JSON",
"RSS listener"
],
"author": "Amaury Joly <amaury.joly@hotmail.com>",
"license": "MIT",
"devDependencies": {
@@ -36,6 +43,6 @@
},
"dependencies": {
"@databases/sqlite": "^3.0.0",
"listener-rss": "file:../listener-rss"
"listener-rss": "^0.0.1"
}
}

20
src/manage-listener.ts
View File

@@ -22,24 +22,24 @@ export class ManageListener extends EventEmitter {
const configs: ListenerRSSInfos[] = await this.sqliteDb.fetchAll();
configs.forEach((config) => {
const newListener = new ListenerRss(config);
this.listenerArray.push(newListener);
this.settingEvents(newListener);
this.addNewListener(config);
});
}
settingEvents(newListener: ListenerRss): void {
private addNewListener(info: ListenerRSSInfos) : ListenerRss {
const newListener = new ListenerRss(info);
this.listenerArray.push(newListener);
newListener.on("update", (obj) => this.emit("update", obj));
newListener.on("newEntries", (obj) => this.emit("newEntries", obj));
newListener.on("error", (err) => this.emit("error", err));
return newListener;
}
async addNewListener(info: ListenerRSSInfos) {
const newListener = new ListenerRss(info);
this.listenerArray.push(newListener);
this.settingEvents(newListener);
await this.sqliteDb.insertListener(newListener);
async registerListener(info: ListenerRSSInfos) {
const listener = this.addNewListener(info);
await this.sqliteDb.insertListener(listener);
}
async save() {

87
src/sqlite-tools.ts
View File

@@ -1,6 +1,6 @@
import { ListenerRSSInfos } from "listener-rss";
import connect, { sql } from "@databases/sqlite";
import connect, { DatabaseConnection, sql } from "@databases/sqlite";
export class SqliteTools {
path?: string;
@@ -9,30 +9,39 @@ export class SqliteTools {
this.path = path;
}
async ensureTableExists() {
async withDB<Type>(callback: (db: DatabaseConnection) => Promise<Type>): Promise<Type> {
const db = connect(this.path);
try {
return callback(db);
} finally {
await db.dispose();
}
}
let req = sql`CREATE TABLE IF NOT EXISTS listeners
(
address TEXT NOT NULL UNIQUE,
customfields TEXT DEFAULT '[]' NOT NULL,
timeloop INTEGER DEFAULT 300 NOT NULL,
last_entries_links TEXT DEFAULT '[]' NOT NULL,
PRIMARY KEY (address),
CHECK(timeloop >= 0)
);`;
await db.query(req);
await db.dispose();
async ensureTableExists() {
await this.withDB(async (db) => {
let req = sql`CREATE TABLE IF NOT EXISTS listeners
(
address TEXT NOT NULL,
customfields TEXT DEFAULT '[]' NOT NULL,
timeloop INTEGER DEFAULT 300 NOT NULL,
last_entries_links TEXT DEFAULT '[]' NOT NULL,
PRIMARY KEY (address),
CHECK(timeloop >= 0)
);`;
await db.query(req);
});
}
async fetchAll(): Promise<ListenerRSSInfos[]> {
const db = connect(this.path);
const rows = await this.withDB(async (db) => {
let req = sql`SELECT *
FROM listeners`;
return await db.query(req);
});
let req = sql`SELECT *
FROM listeners`;
const rows = await db.query(req);
await db.dispose();
return rows.map((row: any) => ({
address: row["address"],
customfields: JSON.parse(row["customfields"]),
@@ -42,30 +51,28 @@ export class SqliteTools {
}
async insertListener(listener: ListenerRSSInfos) {
const db = connect(this.path);
let req = sql`INSERT INTO listeners (address, timeloop, customfields, last_entries_links)
VALUES (${listener.address},
${listener.timeloop},
${JSON.stringify(listener.customfields ? listener.customfields : [])},
${JSON.stringify(listener.lastEntriesLinks ? listener.lastEntriesLinks : [])})`;
await db.query(req);
await db.dispose();
await this.withDB(async (db) => {
let req = sql`INSERT INTO listeners (address, timeloop, customfields, last_entries_links)
VALUES (${listener.address},
${listener.timeloop},
${JSON.stringify(listener.customfields ?? [])},
${JSON.stringify(listener.lastEntriesLinks ?? [])})`;
await db.query(req);
});
}
async updateAll(listeners: ListenerRSSInfos[]) {
const db = connect(this.path);
await db.tx(async (transaction) => {
for (const listener of listeners) {
let req = sql`UPDATE listeners
SET last_entries_links = ${JSON.stringify(
listener.lastEntriesLinks
)}
WHERE address = ${listener.address}`;
await transaction.query(req);
}
await this.withDB(async (db) => {
await db.tx(async (transaction) => {
for (const listener of listeners) {
let req = sql`UPDATE listeners
SET last_entries_links = ${JSON.stringify(
listener.lastEntriesLinks
)}
WHERE address = ${listener.address}`;
await transaction.query(req);
}
});
});
await db.dispose();
}
}

177
tests/index-spec.ts
View File

@@ -32,7 +32,7 @@ describe("test class ManageListener", function () {
const ml = new ManageListener(path);
await ml.load();
for (const item of dataWithoutHistory) await ml.addNewListener(item);
for (const item of dataWithoutHistory) await ml.registerListener(item);
// when
ml.save();
@@ -62,7 +62,7 @@ describe("test class ManageListener", function () {
const clock = sinon.useFakeTimers();
const ml = new ManageListener(path);
await ml.load();
for (const item of dataWithoutHistory) await ml.addNewListener(item);
for (const item of dataWithoutHistory) await ml.registerListener(item);
const updateSpy = sinon.spy();
const newEntriesSpy = sinon.spy();
@@ -152,7 +152,7 @@ describe("test class ManageListener", function () {
const clock = sinon.useFakeTimers();
const ml = new ManageListener(path);
await ml.load();
for (const item of dataWithHistory) await ml.addNewListener(item);
for (const item of dataWithHistory) await ml.registerListener(item);
const updateSpy = sinon.spy();
const newEntriesSpy = sinon.spy();
@@ -251,176 +251,5 @@ describe("test class ManageListener", function () {
ml.stopAll();
});
});
// it("should call correctly the events (with aggregation)", async function () {
// // given
// const clock = sinon.useFakeTimers();
// const ml = new ManageListener({
// timeloop: 30,
// path: "tests/RessourcesTest/RealRessources/save_no_history.json",
// });
// const updateSpy = sinon.spy((obj) => console.log(obj));
// const newEntriesSpy = sinon.spy();
// const tabChannelId = [
// "UCOuIgj0CYCXCvjWywjDbauw",
// "UCh2YBKhYIy-_LtfCIn2Jycg",
// "UCY7klexd1qEqxgqYK6W7BVQ",
// ];
// ml.on("update", updateSpy);
// ml.on("newEntries", newEntriesSpy);
// tabChannelId.forEach((item: string) => {
// nock("https://www.youtube.com")
// .get(`/feeds/videos.xml?channel_id=${item}`)
// .replyWithFile(
// 200,
// path.join(__dirname, `RessourcesTest/RealRessources/${item}.rss`),
// { "content-type": "text/xml", charset: "utf-8" }
// )
// .persist();
// });
// ml.startAll();
// // when
// await Promise.all([
// events.once(ml.listenerArray[0], "update"),
// events.once(ml.listenerArray[1], "update"),
// events.once(ml.listenerArray[2], "update"),
// ]);
// //expect
// expect(updateSpy).to.have.been.calledOnce;
// expect(newEntriesSpy).to.have.been.calledOnce;
// // given
// updateSpy.resetHistory();
// newEntriesSpy.resetHistory();
// // when
// await clock.tickAsync(10000);
// await events.once(ml.listenerArray[0], "update");
// //expect
// expect(updateSpy).to.not.have.been.called;
// expect(newEntriesSpy).to.not.have.been.called;
// // when
// await clock.tickAsync(10000);
// await Promise.all([
// events.once(ml.listenerArray[0], "update"),
// events.once(ml.listenerArray[1], "update"),
// ]);
// //expect
// expect(updateSpy).to.not.have.been.called;
// expect(newEntriesSpy).to.not.have.been.called;
// // when
// await clock.tickAsync(10000);
// await Promise.all([
// events.once(ml.listenerArray[0], "update"),
// events.once(ml.listenerArray[2], "update"),
// ]);
// //expect
// expect(updateSpy).to.have.been.calledOnce;
// expect(newEntriesSpy).to.not.have.been.called;
// ml.stopAll();
// });
// it("should call correctly the events with a new entry (with aggregation)", async function () {
// // given
// const clock = sinon.useFakeTimers();
// const ml = new ManageListener({
// timeloop: 30,
// path: "tests/RessourcesTest/RealRessources/save.json",
// });
// const updateSpy = sinon.spy();
// const newEntriesSpy = sinon.spy();
// const tabChannelId = [
// "UCOuIgj0CYCXCvjWywjDbauw",
// "UCh2YBKhYIy-_LtfCIn2Jycg",
// "UCY7klexd1qEqxgqYK6W7BVQ",
// ];
// ml.on("update", updateSpy);
// ml.on("newEntries", newEntriesSpy);
// tabChannelId.forEach((item: string) => {
// nock("https://www.youtube.com")
// .get(`/feeds/videos.xml?channel_id=${item}`)
// .once()
// .replyWithFile(
// 200,
// path.join(__dirname, `RessourcesTest/RealRessources/${item}.rss`),
// { "content-type": "text/xml", charset: "utf-8" }
// );
// });
// ml.startAll();
// // when
// await Promise.all([
// events.once(ml.listenerArray[0], "update"),
// events.once(ml.listenerArray[1], "update"),
// events.once(ml.listenerArray[2], "update"),
// ]);
// //expect
// expect(updateSpy).to.have.been.calledOnce;
// expect(newEntriesSpy).to.not.have.been.called;
// // given
// updateSpy.resetHistory();
// tabChannelId.forEach((item: string) => {
// nock("https://www.youtube.com")
// .get(`/feeds/videos.xml?channel_id=${item}`)
// .replyWithFile(
// 200,
// path.join(
// __dirname,
// `RessourcesTest/RealRessources/WithUpdate/${item}.rss`
// ),
// { "content-type": "text/xml", charset: "utf-8" }
// )
// .persist();
// });
// // when
// await clock.tickAsync(10000);
// await events.once(ml.listenerArray[0], "update");
// //expect
// expect(updateSpy).to.not.have.been.called;
// expect(newEntriesSpy).to.not.have.been.called;
// // when
// await clock.tickAsync(10000);
// await Promise.all([
// events.once(ml.listenerArray[0], "update"),
// events.once(ml.listenerArray[1], "update"),
// ]);
// //expect
// expect(updateSpy).to.not.have.been.called;
// expect(newEntriesSpy).to.not.have.been.called;
// // when
// await clock.tickAsync(10000);
// await Promise.all([
// events.once(ml.listenerArray[0], "update"),
// events.once(ml.listenerArray[2], "update"),
// ]);
// //expect
// expect(updateSpy).to.have.been.calledOnce;
// expect(newEntriesSpy).to.have.been.calledOnce;
// ml.stopAll();
// });
});
});