From b3189e3bf4104e470460aa27e2f5b57ee37451ea Mon Sep 17 00:00:00 2001 From: pikiou Date: Mon, 31 Jan 2022 13:41:34 +0100 Subject: [PATCH] Add generating anonymized db for new devs --- .gitignore | 5 +- src/server/gsheets/accessors.ts | 68 ++------ src/server/gsheets/localDb.ts | 299 ++++++++++++++++++++++++++++++++ 3 files changed, 316 insertions(+), 56 deletions(-) create mode 100644 src/server/gsheets/localDb.ts diff --git a/.gitignore b/.gitignore index 85d1a9a..3db9f04 100644 --- a/.gitignore +++ b/.gitignore @@ -12,10 +12,7 @@ public/* !public/manifest.json # Access -access/gsheets.json -access/jwt_secret.json -access/db.json -access/dbToLoad.json +access/* # Misc .DS_Store diff --git a/src/server/gsheets/accessors.ts b/src/server/gsheets/accessors.ts index 81d34c1..972c78d 100644 --- a/src/server/gsheets/accessors.ts +++ b/src/server/gsheets/accessors.ts @@ -3,35 +3,21 @@ import path from "path" import _ from "lodash" import { promises as fs, constants } from "fs" import { GoogleSpreadsheet, GoogleSpreadsheetWorksheet } from "google-spreadsheet" +import { SheetNames, saveLocalDb, loadLocalDb } from "./localDb" + +export { SheetNames } from "./localDb" // Test write attack with: wget --header='Content-Type:application/json' --post-data='{"prenom":"Pierre","nom":"SCELLES","email":"test@gmail.com","telephone":"0601010101","dejaBenevole":false,"commentaire":""}' http://localhost:3000/PreVolunteerAdd const CRED_PATH = path.resolve(process.cwd(), "access/gsheets.json") -const DB_PATH = path.resolve(process.cwd(), "access/db.json") -const DB_TO_LOAD_PATH = path.resolve(process.cwd(), "access/dbToLoad.json") const REMOTE_UPDATE_DELAY = 40000 const DELAY_AFTER_QUERY = 2000 let creds: string | undefined | null -// eslint-disable-next-line @typescript-eslint/ban-types -let states: { [sheetName in keyof SheetNames]?: object[] | undefined } = {} -// eslint-disable-next-line @typescript-eslint/ban-types -let types: { [sheetName in keyof SheetNames]?: object | undefined } = {} export type ElementWithId = { id: number } & ElementNoId -export class SheetNames { - JavGames = "Jeux JAV" - - PreVolunteers = "PreMembres" - - Teams = "Equipes" - - Volunteers = "Membres" - - Wishes = "Envies d'aider" -} export const sheetNames = new SheetNames() // eslint-disable-next-line @typescript-eslint/ban-types @@ -55,7 +41,7 @@ export async function hasGSheetsAccess(): Promise { export async function checkGSheetsAccess(): Promise { if (!(await hasGSheetsAccess())) { - console.error(`Google Sheets: no creds found, loading local database ${DB_PATH} instead`) + console.error(`Google Sheets: no creds found, loading local database instead`) } } export function getSheet< @@ -126,7 +112,7 @@ export class Sheet< async setList(newState: Element[] | undefined): Promise { this._state = JSON.parse(JSON.stringify(newState)) this.modifiedSinceSave = true - this.localDbSave() + this.saveLocalDb() } async nextId(): Promise { @@ -182,41 +168,16 @@ export class Sheet< this.dbLoad() } } - if (__DEV__) { - this.localDbSave() - } } - async localDbSave(): Promise { - states[this.name] = this._state - types[this.name] = this._type - const toSave = { states, types } - const jsonDB = __DEV__ ? JSON.stringify(toSave, null, 2) : JSON.stringify(toSave) - await fs.writeFile(DB_PATH, jsonDB) + async saveLocalDb(): Promise { + await saveLocalDb(this.name, this._state, this._type) } - async localDbLoad(): Promise { - if (_.isEmpty(states)) { - let stringifiedDb - try { - stringifiedDb = await fs.readFile(DB_TO_LOAD_PATH) - } catch { - console.error(`No local database save found in ${DB_TO_LOAD_PATH}`) - process.exit() - } - if (stringifiedDb) { - const db = JSON.parse(stringifiedDb.toString()) - states = db.states - types = db.types - } - } - - if (!states[this.name]) { - console.error(`Sheet ${this.name} couldn't be found in localDb`) - process.exit() - } - this._state = states[this.name] as Element[] - this._type = types[this.name] as Record + async loadLocalDb(): Promise { + const db = await loadLocalDb(this.name) + this._state = db.state as Element[] + this._type = db.type as Record } dbSave(): void { @@ -250,10 +211,13 @@ export class Sheet< } async dbFirstLoad(): Promise { - if (!(await hasGSheetsAccess()) && _.isEmpty(states)) { - this.localDbLoad() + if (!(await hasGSheetsAccess())) { + await this.loadLocalDb() } this.dbLoad() + if (__DEV__ && (await hasGSheetsAccess())) { + this.saveLocalDb() + } } private async dbSaveAsync(): Promise { diff --git a/src/server/gsheets/localDb.ts b/src/server/gsheets/localDb.ts new file mode 100644 index 0000000..93a2c25 --- /dev/null +++ b/src/server/gsheets/localDb.ts @@ -0,0 +1,299 @@ +// eslint-disable-next-line max-classes-per-file +import path from "path" +import _ from "lodash" +import { promises as fs } from "fs" +import { Volunteer } from "../../services/volunteers" +import { PreVolunteer } from "../../services/preVolunteers" + +const DB_PATH = path.resolve(process.cwd(), "access/db.json") +const DB_TO_LOAD_PATH = path.resolve(process.cwd(), "access/dbToLoad.json") +const ANONYMIZED_DB_PATH = path.resolve(process.cwd(), "access/dbAnonymized.json") + +export class SheetNames { + JavGames = "Jeux JAV" + + PreVolunteers = "PreMembres" + + Teams = "Equipes" + + Volunteers = "Membres" + + Wishes = "Envies d'aider" +} + +// eslint-disable-next-line @typescript-eslint/ban-types +type States = { [sheetName in keyof SheetNames]?: object[] | undefined } +let states: States = {} +// eslint-disable-next-line @typescript-eslint/ban-types +type Types = { [sheetName in keyof SheetNames]?: object | undefined } +let types: Types = {} + +export async function saveLocalDb( + name: keyof SheetNames, + // eslint-disable-next-line @typescript-eslint/ban-types + state: object[] | undefined, + // eslint-disable-next-line @typescript-eslint/ban-types + type: object | undefined +): Promise { + states[name] = state + types[name] = type + const toSave = { states, types } + const jsonDB = __DEV__ ? JSON.stringify(toSave, null, 2) : JSON.stringify(toSave) + await fs.writeFile(DB_PATH, jsonDB) + + toSave.states = anonimizedDb(toSave.states) + const jsonAnonimizedDB = __DEV__ ? JSON.stringify(toSave, null, 2) : JSON.stringify(toSave) + await fs.writeFile(ANONYMIZED_DB_PATH, jsonAnonimizedDB) +} + +export async function loadLocalDb( + name: keyof SheetNames + // eslint-disable-next-line @typescript-eslint/ban-types +): Promise<{ state: object[]; type: object }> { + if (_.isEmpty(states)) { + let stringifiedDb + try { + stringifiedDb = await fs.readFile(DB_TO_LOAD_PATH) + } catch { + console.error(`No local database save found in ${DB_TO_LOAD_PATH}`) + process.exit() + } + if (stringifiedDb) { + const db = JSON.parse(stringifiedDb.toString()) + states = db.states + types = db.types + } + } + + if (!states[name]) { + console.error(`Sheet ${name} couldn't be found in localDb`) + process.exit() + } + return { + state: states[name] as Element[], + type: types[name] as Record, + } +} + +export function isDbLoaded(): boolean { + return !_.isEmpty(states) +} + +const fakeFirstnames = [ + "Suzanne", + "Louis", + "Madeleine", + "Paul", + "Alain", + "Sylvie", + "Pierre", + "Daniel", + "Jacques", + "Martine", + "Anne", + "Bernard", + "Claude", + "Louise", + "René", + "Jacqueline", + "Françoise", + "Christian", + "Nathalie", + "Nicole", + "Isabelle", + "Monique", + "Denise", + "Marie", + "Jeanne", + "Georges", + "Christine", + "Nicolas", + "Michel", + "Christiane", + "Jean", + "Marcel", + "Marguerite", + "André", + "Hélène", + "Henri", + "Roger", + "Catherine", + "Philippe", + "Robert", +] +const fakeLastnames = [ + "Bernard", + "Robert", + "Guerin", + "Joly", + "Dumont", + "Robin", + "Vincent", + "Perrin", + "Picard", + "Lefevre", + "Roy", + "Martin", + "Barbier", + "Clement", + "Lucas", + "Gauthier", + "Duval", + "Bertrand", + "Roux", + "Girard", + "Dupont", + "Leroy", + "Dufour", + "Muller", + "Martinez", + "Thomas", + "Durand", + "Dubois", + "Marie", + "Marchand", + "Lefebvre", + "Brun", + "Bonnet", + "Moreau", + "Francois", + "Gerard", + "Rousseau", + "Faure", + "Renard", + "Meunier", + "David", + "Giraud", + "Simon", + "Vidal", + "Fournier", + "Arnaud", + "Blanchard", + "Colin", + "Meyer", + "Leroux", + "Lemaire", + "Mercier", + "Garnier", + "Morin", + "Lambert", + "Fontaine", + "Morel", + "Michel", + "Blanc", + "Denis", + "Noel", + "Roussel", + "Riviere", + "Garcia", + "Schmitt", + "Chevalier", + "Mathieu", + "Petit", + "Gautier", + "Gaillard", + "Legrand", + "Henry", + "Laurent", + "Caron", + "Andre", + "Masson", + "Nicolas", + "Roche", + "Richard", + "Brunet", + "Boyer", +] +const fakeEmailDomains = [ + "yahoo.fr", + "gmail.com", + "gmail.com", + "gmail.com", + "gmail.com", + "gmail.com", + "gmail.com", + "noos.fr", + "hotmail.com", + "hotmail.com", + "hotmail.com", + "live.fr", + "hotmail.fr", + "wanadoo.fr", + "free.fr", + "yahoo.com", + "laposte.net", + "outlook.com", + "gmail", + "mailoo.org", + "padaone.fr", + "orange.fr", + "netcourrier.com", + "neuf.fr", + "poptalks.com", + "GMAIL.COM", + "wethinkcode.co.za", + "aliceadsl.fr", + "inextenso.fr", + "aol.com", + "epitech.eu", + "me.com", + "sncf.fr", + "outlook.fr", + "paris.fr", + "sfr.fr", + "lilo.org", + "protonmail.com", + "posteo.net", + "msn.com", + "mailo.org", + "live.com", +] + +function anonimizedDb(_s: States): States { + const s = _.cloneDeep(_s) + if (s.Volunteers) { + ;(s.Volunteers as Volunteer[]).forEach((v) => { + v.firstname = fakeFirstnames[numberToRand(v.id) % fakeFirstnames.length] + v.lastname = fakeLastnames[numberToRand(v.id) % fakeLastnames.length] + const fakeEmailDomain = fakeEmailDomains[numberToRand(v.id) % fakeEmailDomains.length] + v.email = `${v.firstname}.${v.lastname}.${v.id}@${fakeEmailDomain}`.toLowerCase() + const mobileStart = v.mobile.match(/^\+?[0-9][0-9]/) + const mobileEnd = [1, 2, 3, 4] + .map((n) => `${numberToRand(v.id + n) % 10}${numberToRand(v.id + n + 10) % 10}`) + .join(" ") + v.mobile = v.mobile ? `${(mobileStart || ["06"])[0]} ${mobileEnd}` : "" + v.photo = `${v.firstname}_${v.lastname}.jpg`.toLowerCase() + v.password1 = "$2y$1a$Kt/FAKEFAKEFAKEFAKEc6.FAKEFAKEFAKEFAKE//FAKEFAKEFAKEy" + v.password2 = "$2y$1a$Kt/FAKEFAKEFAKEFAKEc6.FAKEFAKEFAKEFAKE//FAKEFAKEFAKEy" + v.acceptsNotifs = "" + if (v.id % 13 === 0) { + v.acceptsNotifs = "oui" + } else if (v.id % 251 === 0) { + v.acceptsNotifs = "non" + } + v.pushNotifSubscription = + v.id % 13 === 0 + ? '{"endpoint":"https://fcm.googleapis.com/fcm/send/f-EAfakedfakedU:APA91fakedfakedzIk-DEglfakedfaked9ugI--ljtfakedfakedfakedfakedfakedfakedP3t-ggU7Afakedfakedfakedkai","expirationTime":null,"keys":{"p256dh":"BEZOJSfakedfakedfakedfakedfakedfakedfakedfakedfakedfakedgYs-cafakedw","auth":"GlMfakedfakedFRg"}}' + : "" + }) + } + if (s.PreVolunteers) { + ;(s.PreVolunteers as PreVolunteer[]).forEach((v) => { + v.firstname = fakeFirstnames[numberToRand(v.id) % fakeFirstnames.length] + v.lastname = fakeLastnames[numberToRand(v.id) % fakeLastnames.length] + const fakeEmailDomain = fakeEmailDomains[numberToRand(v.id) % fakeEmailDomains.length] + v.email = `${v.firstname}.${v.lastname}.${v.id}@${fakeEmailDomain}`.toLowerCase() + const mobileStart = v.mobile.match(/^\+?[0-9][0-9]/) + const mobileEnd = [1, 2, 3, 4] + .map((n) => `${numberToRand(v.id + n) % 10}${numberToRand(v.id + n + 10) % 10}`) + .join(" ") + v.mobile = v.mobile ? `${(mobileStart || ["06"])[0]} ${mobileEnd}` : "" + v.comment = v.id % 3 === 0 ? "Bonjour, j'adore l'initiative!" : "" + }) + } + return s +} + +function numberToRand(n: number) { + return (1664525 * n + 1013904223) % 512 +}