diff --git a/gsheetsTest.ts b/gsheetsTest.ts deleted file mode 100644 index cdc1181..0000000 --- a/gsheetsTest.ts +++ /dev/null @@ -1,406 +0,0 @@ -/* eslint-disable max-classes-per-file */ -import * as _ from "lodash" -import path from "path" -import { promises as fs } from "fs" -import { GoogleSpreadsheet, GoogleSpreadsheetWorksheet } from "google-spreadsheet" - -const SCOPES = ["https://www.googleapis.com/auth/spreadsheets"] -const CRED_PATH = path.resolve(process.cwd(), "./access/gsheets.json") - -// eslint-disable-next-line @typescript-eslint/ban-types -export async function getList( - sheetName: string, - specimen: Element -): Promise { - type StringifiedElement = Record - const sheet = await getGSheet(sheetName) - - // Load sheet into an array of objects - const rows = (await sheet.getRows()) as StringifiedElement[] - const elements: Element[] = [] - if (!rows[0]) { - // TODO: Report format error to database maintainers - return [] - } - const types = _.pick(rows[0], Object.keys(specimen)) as Record - rows.shift() - rows.forEach((row) => { - const stringifiedElement = _.pick(row, Object.keys(specimen)) as Record< - keyof Element, - string - > - const element = parseElement(stringifiedElement, types, specimen) - if (element !== undefined) { - elements.push(element) - } - }) - - return elements -} - -// eslint-disable-next-line @typescript-eslint/ban-types -export async function setList( - sheetName: string, - elements: Element[] -): Promise { - const sheet = await getGSheet(sheetName) - - // Load sheet into an array of objects - const rows = await sheet.getRows() - if (!rows[0]) { - return undefined - } - const types = _.pick(rows[0], Object.keys(elements[0] || {})) as Record - - // Update received rows - let rowid = 1 - // eslint-disable-next-line no-restricted-syntax - for (const element of elements) { - const row = rows[rowid] - const stringifiedRow = stringifyElement(element, types) - - if (stringifiedRow === undefined) { - return undefined - } - - if (!row) { - // eslint-disable-next-line no-await-in-loop - await sheet.addRow(stringifiedRow) - } else { - const keys = Object.keys(stringifiedRow) - const sameCells = _.every( - keys, - (key: keyof Element) => row[key as string] === stringifiedRow[key] - ) - if (!sameCells) { - keys.forEach((key) => { - row[key] = stringifiedRow[key as keyof Element] - }) - // eslint-disable-next-line no-await-in-loop - await row.save() - } - } - - rowid += 1 - } - - // Delete all following rows - for (let rowToDelete = sheet.rowCount - 1; rowToDelete >= rowid; rowToDelete -= 1) { - if (rows[rowToDelete]) { - // eslint-disable-next-line no-await-in-loop - await rows[rowToDelete].delete() - } - } - - return true -} - -async function getGSheet(sheetName: string): Promise { - const doc = new GoogleSpreadsheet("1pMMKcYx6NXLOqNn6pLHJTPMTOLRYZmSNg2QQcAu7-Pw") - const creds = await fs.readFile(CRED_PATH) - // Authentication - await doc.useServiceAccountAuth(JSON.parse(creds.toString())) - await doc.loadInfo() - return doc.sheetsByTitle[sheetName] -} - -// eslint-disable-next-line @typescript-eslint/ban-types -function parseElement( - rawElement: Record, - types: Record, - specimen: Element -): Element | undefined { - const fullElement = _.reduce( - types, - (element: any, type: string, prop: string) => { - if (element === undefined) { - return undefined - } - const rawProp: string = rawElement[prop as keyof Element] - switch (type) { - case "string": - element[prop] = rawProp - break - - case "number": - element[prop] = +rawProp - break - - case "boolean": - element[prop] = rawProp !== "0" && rawProp !== "" - break - - case "date": - // eslint-disable-next-line no-case-declarations - const matchDate = rawProp.match(/^([0-9]+)\/([0-9]+)\/([0-9]+)$/) - if (matchDate) { - element[prop] = new Date(+matchDate[3], +matchDate[2] - 1, +matchDate[1]) - break - } - return undefined // TODO: Report format error to database maintainers - break - - default: - // eslint-disable-next-line no-case-declarations - const matchArrayType = type.match(/^(number|string|boolean|date)\[([^\]]+)\]$/) - if (!matchArrayType) { - return undefined - } - if (!rawProp) { - element[prop] = [] - } else { - const arrayType = matchArrayType[1] - const delimiter = matchArrayType[2] - - switch (arrayType) { - case "string": - element[prop] = rawProp.split(delimiter) - break - - case "number": - element[prop] = _.map(rawProp.split(delimiter), (val) => +val) - break - - case "boolean": - element[prop] = _.map( - rawProp.split(delimiter), - (val) => val !== "0" && val !== "" - ) - break - - case "date": - // eslint-disable-next-line no-case-declarations - const rawDates = rawProp.split(delimiter) - element[prop] = [] - // eslint-disable-next-line no-case-declarations - const rightFormat = rawDates.every((rawDate) => { - const matchDateArray = rawDate.match( - /^([0-9]+)\/([0-9]+)\/([0-9]+)$/ - ) - if (!matchDateArray) { - return false - } - element[prop].push( - new Date( - +matchDateArray[3], - +matchDateArray[2] - 1, - +matchDateArray[1] - ) - ) - return true - }) - if (!rightFormat) { - return undefined - } - break - default: - } - } - } - return element - }, - JSON.parse(JSON.stringify(specimen)) - ) - return fullElement -} - -// eslint-disable-next-line @typescript-eslint/ban-types -function stringifyElement( - element: Element, - types: Record -): Record | undefined { - const rawElement: Record | undefined = _.reduce( - types, - ( - stringifiedElement: Record | undefined, - type: string, - prop: string - ) => { - if (stringifiedElement === undefined) { - return undefined - } - const value = element[prop as keyof Element] - switch (type) { - case "string": - stringifiedElement[prop as keyof Element] = formulaSafe(`${value}`) - break - - case "number": - stringifiedElement[prop as keyof Element] = `${value}` - break - - case "boolean": - stringifiedElement[prop as keyof Element] = value ? "X" : "" - break - - case "date": - if (value instanceof Date) { - stringifiedElement[prop as keyof Element] = `${value.getDate()}/${ - value.getMonth() + 1 - }/${value.getFullYear()}` - break - } else { - console.error("Wrong date format in stringifyElement") - return undefined // TODO: Report format error to database maintainers - } - - default: - // eslint-disable-next-line no-case-declarations - const matchArrayType = type.match(/^(number|string|boolean|date)\[([^\]]+)\]$/) - if (!matchArrayType || !_.isArray(value)) { - console.error("Unknown matchArrayType or not an array in stringifyElement") - return undefined - } - // eslint-disable-next-line no-case-declarations - const arrayType = matchArrayType[1] - // eslint-disable-next-line no-case-declarations - const delimiter = matchArrayType[2] - - switch (arrayType) { - case "string": - if (!_.every(value, _.isString)) { - return undefined - } - stringifiedElement[prop as keyof Element] = formulaSafe( - value.join(delimiter) - ) - break - - case "number": - if (!_.every(value, _.isNumber)) { - return undefined - } - stringifiedElement[prop as keyof Element] = value.join(delimiter) - break - - case "boolean": - if (!_.every(value, _.isBoolean)) { - return undefined - } - stringifiedElement[prop as keyof Element] = _.map(value, (val) => - val ? "X" : "" - ).join(delimiter) - break - - case "date": - if (!_.every(value, _.isDate)) { - return undefined - } - stringifiedElement[prop as keyof Element] = _.map( - value, - (val) => - `${val.getDate()}/${val.getMonth() + 1}/${val.getFullYear()}` - ).join(delimiter) - break - - default: - return undefined - } - } - - return stringifiedElement - }, - JSON.parse(JSON.stringify(element)) - ) - - return rawElement -} - -function formulaSafe(value: string): string { - return value.replace(/^=+/, "") -} - -export { SCOPES } - -class Test { - id = 5 - - wishes = "" - - dateAjout: Date = new Date(0) - - ignore = false - - volunteers: number[] = [] - - equipes: string[] = [] - - datesPossibles: Date[] = [] - - tictactoe: boolean[] = [] -} - -// Can't run it on every test, it requires private access to a google sheet -async function testGSheetAPi(): Promise { - const dataset: Test[] = [ - { - id: 1, - wishes: "Présenter le festival et son organisation à un nouveau bénévol au téléphone", - dateAjout: new Date("2021-10-18T22:00:00.000Z"), - ignore: true, - volunteers: [2, 5, 6, 4, 2, 7], - equipes: ["Accueillir les bénévoles"], - datesPossibles: [ - new Date("2021-11-18T23:00:00.000Z"), - new Date("2021-11-19T23:00:00.000Z"), - new Date("2021-11-20T23:00:00.000Z"), - ], - tictactoe: [true, false, true, false, false, true], - }, - { - id: 5, - wishes: "Créer de jolies pages webs", - dateAjout: new Date("2021-10-18T22:00:00.000Z"), - ignore: false, - volunteers: [7], - equipes: ["Site Web Public", "Force Orange"], - datesPossibles: [], - tictactoe: [], - }, - { - id: 6, - wishes: "Modérer un salon Discord", - dateAjout: new Date("2021-10-18T22:00:00.000Z"), - ignore: true, - volunteers: [], - equipes: [], - datesPossibles: [new Date("2024-10-18T22:00:00.000Z")], - tictactoe: [false, false, false, false, true, true, true, true], - }, - ] - - // console.log("Lecture des Volunteers...") - // const datasetVolunteersLu = await getList("Volunteers", new Volunteer()) - // if (!datasetVolunteersLu) { - // console.log("ECHEC de la lecture des volunteers", datasetVolunteersLu) - // return - // } - // console.log("Extraction des volunteers réussie") - // await fs.writeFile("volunteers.json", JSON.stringify(datasetVolunteersLu)) - - console.log("Test d'écriture...") - const resultatEcriture = await setList("Tests de l'API", dataset) - if (!resultatEcriture) { - console.log("ECHEC de l'écriture") - return - } - console.log("Écriture réussie") - - console.log("Test de lecture...") - const datasetLu = await getList("Tests de l'API", new Test()) - if (!_.isEqual(datasetLu, dataset)) { - console.log("ECHEC de la lecture", datasetLu, dataset) - return - } - console.log("Lecture réussie") - - console.log("Effacement des données...") - const resultatEffacement = await setList("Tests de l'API", []) - if (!resultatEffacement) { - console.log("ECHEC de l'effacement") - return - } - console.log("Effacement réussi") -} - -testGSheetAPi().then(() => console.log("Done")) diff --git a/src/server/gsheets/accessors.ts b/src/server/gsheets/accessors.ts index d777ad8..f0e3439 100644 --- a/src/server/gsheets/accessors.ts +++ b/src/server/gsheets/accessors.ts @@ -9,7 +9,7 @@ import { GoogleSpreadsheet, GoogleSpreadsheetWorksheet } from "google-spreadshee const CRED_PATH = path.resolve(process.cwd(), "access/gsheets.json") const REMOTE_UPDATE_DELAY = 40000 -const DELAY_AFTER_QUERY = 1000 +const DELAY_AFTER_QUERY = 2000 export type ElementWithId = { id: number } & ElementNoId @@ -77,7 +77,7 @@ export class Sheet< readonly translation: { [k in keyof Element]: string } ) { this.invertedTranslation = _.invert(this.translation) - this.sheetName = sheetNames[name] + this.sheetName = sheetNames[name] || name this.frenchSpecimen = _.mapValues( _.invert(translation), (englishProp: string) => (specimen as any)[englishProp] @@ -150,20 +150,28 @@ export class Sheet< } dbSave(): void { - this.modifiedSinceSave = false this.saveTimestamp = +new Date() - this.dbSaveAsync() + try { + this.dbSaveAsync() + this.modifiedSinceSave = false + } catch (e) { + console.error("Error in dbSave: ", e) + } } dbLoad(): void { - this.toRunAfterLoad = [] - this.dbLoadAsync().then(() => { - if (this.toRunAfterLoad) { - this.toRunAfterLoad.map((func) => func()) - this.toRunAfterLoad = undefined - } - }) + try { + this.toRunAfterLoad = [] + this.dbLoadAsync().then(() => { + if (this.toRunAfterLoad) { + this.toRunAfterLoad.map((func) => func()) + this.toRunAfterLoad = undefined + } + }) + } catch (e) { + console.error("Error in dbLoad: ", e) + } } private async dbSaveAsync(): Promise { @@ -174,6 +182,7 @@ export class Sheet< // Load sheet into an array of objects const rows = await sheet.getRows() + await delayDBAccess() if (!rows[0]) { throw new Error(`No column types defined in sheet ${this.name}`) } @@ -234,7 +243,6 @@ export class Sheet< private async dbLoadAsync(): Promise { type StringifiedElement = Record - const sheet = await this.getGSheet() // Load sheet into an array of objects diff --git a/src/utils/standardization.ts b/src/utils/standardization.ts index bb4668d..d200952 100644 --- a/src/utils/standardization.ts +++ b/src/utils/standardization.ts @@ -1,5 +1,5 @@ export function canonicalEmail(email: string): string { - email = email.replace(/^\s+|\s+$/g, "") + email = email.replace(/^\s+|\s+$/g, "").replace(/\+[^@]+/, "") // Remove +pel in pierre+pel@gmail.com if (/@gmail.com$/.test(email)) { let domain = email.replace(/^.*@/, "") domain = domain.replace(/^googlemail%.com$/, "gmail.com") @@ -28,7 +28,7 @@ export function canonicalMobile(mobile: string): string { if (!validMobile(mobile)) { return "" } - let clean = trim(mobile).replace(/[-0-9. ()+]$/g, "") + let clean = trim(mobile).replace(/[-. ()+]/g, "") if (clean.length === 11) { clean = clean.replace(/^33/, "0") } @@ -44,3 +44,30 @@ export function canonicalMobile(mobile: string): string { export function trim(src: string): string { return typeof src !== "string" ? "" : src.replace(/^\s*/, "").replace(/\s*$/, "") } + +// eslint-disable-next-line @typescript-eslint/ban-types +export function doCleanVolunteer< + Element extends { firstname: string; lastname: string; email: string; mobile: string } +>(vol: Element): void { + if (!validMobile(vol.mobile)) { + vol.mobile = "" + } else { + vol.mobile = canonicalMobile(vol.mobile) + } + + vol.firstname = trim(vol.firstname) + vol.firstname = vol.firstname + .toLowerCase() + .replace(/(?<=^|[\s'-])([a-zA-Z]|[à-ú]|[À-Ú])/gi, (s) => s.toUpperCase()) + .replace(/\b(de|d'|du|le|la)\b/gi, (s) => s.toLowerCase()) + .replace(/\b(d'|l')/gi, (s) => s.toLowerCase()) + + vol.lastname = trim(vol.lastname) + vol.lastname = vol.lastname + .toLowerCase() + .replace(/(?<=^|[\s'-])([a-zA-Z]|[à-ú]|[À-Ú])/gi, (s) => s.toUpperCase()) + .replace(/\b(de|d'|du|le|la)\b/gi, (s) => s.toLowerCase()) + .replace(/\b(d'|l')/gi, (s) => s.toLowerCase()) + + vol.email = canonicalEmail(vol.email) +}