More robust gSheet API connection

gsheetsTest.ts

@@ -1,406 +0,0 @@
-/* eslint-disable max-classes-per-file */
-import * as _ from "lodash"
-import path from "path"
-import { promises as fs } from "fs"
-import { GoogleSpreadsheet, GoogleSpreadsheetWorksheet } from "google-spreadsheet"
-
-const SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]
-const CRED_PATH = path.resolve(process.cwd(), "./access/gsheets.json")
-
-// eslint-disable-next-line @typescript-eslint/ban-types
-export async function getList<Element extends object>(
-    sheetName: string,
-    specimen: Element
-): Promise<Element[]> {
-    type StringifiedElement = Record<keyof Element, string>
-    const sheet = await getGSheet(sheetName)
-
-    // Load sheet into an array of objects
-    const rows = (await sheet.getRows()) as StringifiedElement[]
-    const elements: Element[] = []
-    if (!rows[0]) {
-        // TODO: Report format error to database maintainers
-        return []
-    }
-    const types = _.pick(rows[0], Object.keys(specimen)) as Record<keyof Element, string>
-    rows.shift()
-    rows.forEach((row) => {
-        const stringifiedElement = _.pick(row, Object.keys(specimen)) as Record<
-            keyof Element,
-            string
-        >
-        const element = parseElement<Element>(stringifiedElement, types, specimen)
-        if (element !== undefined) {
-            elements.push(element)
-        }
-    })
-
-    return elements
-}
-
-// eslint-disable-next-line @typescript-eslint/ban-types
-export async function setList<Element extends object>(
-    sheetName: string,
-    elements: Element[]
-): Promise<true | undefined> {
-    const sheet = await getGSheet(sheetName)
-
-    // Load sheet into an array of objects
-    const rows = await sheet.getRows()
-    if (!rows[0]) {
-        return undefined
-    }
-    const types = _.pick(rows[0], Object.keys(elements[0] || {})) as Record<keyof Element, string>
-
-    // Update received rows
-    let rowid = 1
-    // eslint-disable-next-line no-restricted-syntax
-    for (const element of elements) {
-        const row = rows[rowid]
-        const stringifiedRow = stringifyElement(element, types)
-
-        if (stringifiedRow === undefined) {
-            return undefined
-        }
-
-        if (!row) {
-            // eslint-disable-next-line no-await-in-loop
-            await sheet.addRow(stringifiedRow)
-        } else {
-            const keys = Object.keys(stringifiedRow)
-            const sameCells = _.every(
-                keys,
-                (key: keyof Element) => row[key as string] === stringifiedRow[key]
-            )
-            if (!sameCells) {
-                keys.forEach((key) => {
-                    row[key] = stringifiedRow[key as keyof Element]
-                })
-                // eslint-disable-next-line no-await-in-loop
-                await row.save()
-            }
-        }
-
-        rowid += 1
-    }
-
-    // Delete all following rows
-    for (let rowToDelete = sheet.rowCount - 1; rowToDelete >= rowid; rowToDelete -= 1) {
-        if (rows[rowToDelete]) {
-            // eslint-disable-next-line no-await-in-loop
-            await rows[rowToDelete].delete()
-        }
-    }
-
-    return true
-}
-
-async function getGSheet(sheetName: string): Promise<GoogleSpreadsheetWorksheet> {
-    const doc = new GoogleSpreadsheet("1pMMKcYx6NXLOqNn6pLHJTPMTOLRYZmSNg2QQcAu7-Pw")
-    const creds = await fs.readFile(CRED_PATH)
-    // Authentication
-    await doc.useServiceAccountAuth(JSON.parse(creds.toString()))
-    await doc.loadInfo()
-    return doc.sheetsByTitle[sheetName]
-}
-
-// eslint-disable-next-line @typescript-eslint/ban-types
-function parseElement<Element extends object>(
-    rawElement: Record<keyof Element, string>,
-    types: Record<keyof Element, string>,
-    specimen: Element
-): Element | undefined {
-    const fullElement = _.reduce(
-        types,
-        (element: any, type: string, prop: string) => {
-            if (element === undefined) {
-                return undefined
-            }
-            const rawProp: string = rawElement[prop as keyof Element]
-            switch (type) {
-                case "string":
-                    element[prop] = rawProp
-                    break
-
-                case "number":
-                    element[prop] = +rawProp
-                    break
-
-                case "boolean":
-                    element[prop] = rawProp !== "0" && rawProp !== ""
-                    break
-
-                case "date":
-                    // eslint-disable-next-line no-case-declarations
-                    const matchDate = rawProp.match(/^([0-9]+)\/([0-9]+)\/([0-9]+)$/)
-                    if (matchDate) {
-                        element[prop] = new Date(+matchDate[3], +matchDate[2] - 1, +matchDate[1])
-                        break
-                    }
-                    return undefined // TODO: Report format error to database maintainers
-                    break
-
-                default:
-                    // eslint-disable-next-line no-case-declarations
-                    const matchArrayType = type.match(/^(number|string|boolean|date)\[([^\]]+)\]$/)
-                    if (!matchArrayType) {
-                        return undefined
-                    }
-                    if (!rawProp) {
-                        element[prop] = []
-                    } else {
-                        const arrayType = matchArrayType[1]
-                        const delimiter = matchArrayType[2]
-
-                        switch (arrayType) {
-                            case "string":
-                                element[prop] = rawProp.split(delimiter)
-                                break
-
-                            case "number":
-                                element[prop] = _.map(rawProp.split(delimiter), (val) => +val)
-                                break
-
-                            case "boolean":
-                                element[prop] = _.map(
-                                    rawProp.split(delimiter),
-                                    (val) => val !== "0" && val !== ""
-                                )
-                                break
-
-                            case "date":
-                                // eslint-disable-next-line no-case-declarations
-                                const rawDates = rawProp.split(delimiter)
-                                element[prop] = []
-                                // eslint-disable-next-line no-case-declarations
-                                const rightFormat = rawDates.every((rawDate) => {
-                                    const matchDateArray = rawDate.match(
-                                        /^([0-9]+)\/([0-9]+)\/([0-9]+)$/
-                                    )
-                                    if (!matchDateArray) {
-                                        return false
-                                    }
-                                    element[prop].push(
-                                        new Date(
-                                            +matchDateArray[3],
-                                            +matchDateArray[2] - 1,
-                                            +matchDateArray[1]
-                                        )
-                                    )
-                                    return true
-                                })
-                                if (!rightFormat) {
-                                    return undefined
-                                }
-                                break
-                            default:
-                        }
-                    }
-            }
-            return element
-        },
-        JSON.parse(JSON.stringify(specimen))
-    )
-    return fullElement
-}
-
-// eslint-disable-next-line @typescript-eslint/ban-types
-function stringifyElement<Element extends object>(
-    element: Element,
-    types: Record<keyof Element, string>
-): Record<keyof Element, string> | undefined {
-    const rawElement: Record<keyof Element, string> | undefined = _.reduce(
-        types,
-        (
-            stringifiedElement: Record<keyof Element, string> | undefined,
-            type: string,
-            prop: string
-        ) => {
-            if (stringifiedElement === undefined) {
-                return undefined
-            }
-            const value = element[prop as keyof Element]
-            switch (type) {
-                case "string":
-                    stringifiedElement[prop as keyof Element] = formulaSafe(`${value}`)
-                    break
-
-                case "number":
-                    stringifiedElement[prop as keyof Element] = `${value}`
-                    break
-
-                case "boolean":
-                    stringifiedElement[prop as keyof Element] = value ? "X" : ""
-                    break
-
-                case "date":
-                    if (value instanceof Date) {
-                        stringifiedElement[prop as keyof Element] = `${value.getDate()}/${
-                            value.getMonth() + 1
-                        }/${value.getFullYear()}`
-                        break
-                    } else {
-                        console.error("Wrong date format in stringifyElement")
-                        return undefined // TODO: Report format error to database maintainers
-                    }
-
-                default:
-                    // eslint-disable-next-line no-case-declarations
-                    const matchArrayType = type.match(/^(number|string|boolean|date)\[([^\]]+)\]$/)
-                    if (!matchArrayType || !_.isArray(value)) {
-                        console.error("Unknown matchArrayType or not an array in stringifyElement")
-                        return undefined
-                    }
-                    // eslint-disable-next-line no-case-declarations
-                    const arrayType = matchArrayType[1]
-                    // eslint-disable-next-line no-case-declarations
-                    const delimiter = matchArrayType[2]
-
-                    switch (arrayType) {
-                        case "string":
-                            if (!_.every(value, _.isString)) {
-                                return undefined
-                            }
-                            stringifiedElement[prop as keyof Element] = formulaSafe(
-                                value.join(delimiter)
-                            )
-                            break
-
-                        case "number":
-                            if (!_.every(value, _.isNumber)) {
-                                return undefined
-                            }
-                            stringifiedElement[prop as keyof Element] = value.join(delimiter)
-                            break
-
-                        case "boolean":
-                            if (!_.every(value, _.isBoolean)) {
-                                return undefined
-                            }
-                            stringifiedElement[prop as keyof Element] = _.map(value, (val) =>
-                                val ? "X" : ""
-                            ).join(delimiter)
-                            break
-
-                        case "date":
-                            if (!_.every(value, _.isDate)) {
-                                return undefined
-                            }
-                            stringifiedElement[prop as keyof Element] = _.map(
-                                value,
-                                (val) =>
-                                    `${val.getDate()}/${val.getMonth() + 1}/${val.getFullYear()}`
-                            ).join(delimiter)
-                            break
-
-                        default:
-                            return undefined
-                    }
-            }
-
-            return stringifiedElement
-        },
-        JSON.parse(JSON.stringify(element))
-    )
-
-    return rawElement
-}
-
-function formulaSafe(value: string): string {
-    return value.replace(/^=+/, "")
-}
-
-export { SCOPES }
-
-class Test {
-    id = 5
-
-    wishes = ""
-
-    dateAjout: Date = new Date(0)
-
-    ignore = false
-
-    volunteers: number[] = []
-
-    equipes: string[] = []
-
-    datesPossibles: Date[] = []
-
-    tictactoe: boolean[] = []
-}
-
-// Can't run it on every test, it requires private access to a google sheet
-async function testGSheetAPi(): Promise<void> {
-    const dataset: Test[] = [
-        {
-            id: 1,
-            wishes: "Présenter le festival et son organisation à un nouveau bénévol au téléphone",
-            dateAjout: new Date("2021-10-18T22:00:00.000Z"),
-            ignore: true,
-            volunteers: [2, 5, 6, 4, 2, 7],
-            equipes: ["Accueillir les bénévoles"],
-            datesPossibles: [
-                new Date("2021-11-18T23:00:00.000Z"),
-                new Date("2021-11-19T23:00:00.000Z"),
-                new Date("2021-11-20T23:00:00.000Z"),
-            ],
-            tictactoe: [true, false, true, false, false, true],
-        },
-        {
-            id: 5,
-            wishes: "Créer de jolies pages webs",
-            dateAjout: new Date("2021-10-18T22:00:00.000Z"),
-            ignore: false,
-            volunteers: [7],
-            equipes: ["Site Web Public", "Force Orange"],
-            datesPossibles: [],
-            tictactoe: [],
-        },
-        {
-            id: 6,
-            wishes: "Modérer un salon Discord",
-            dateAjout: new Date("2021-10-18T22:00:00.000Z"),
-            ignore: true,
-            volunteers: [],
-            equipes: [],
-            datesPossibles: [new Date("2024-10-18T22:00:00.000Z")],
-            tictactoe: [false, false, false, false, true, true, true, true],
-        },
-    ]
-
-    // console.log("Lecture des Volunteers...")
-    // const datasetVolunteersLu = await getList<Volunteer>("Volunteers", new Volunteer())
-    // if (!datasetVolunteersLu) {
-    //     console.log("ECHEC de la lecture des volunteers", datasetVolunteersLu)
-    //     return
-    // }
-    // console.log("Extraction des volunteers réussie")
-    // await fs.writeFile("volunteers.json", JSON.stringify(datasetVolunteersLu))
-
-    console.log("Test d'écriture...")
-    const resultatEcriture = await setList<Test>("Tests de l'API", dataset)
-    if (!resultatEcriture) {
-        console.log("ECHEC de l'écriture")
-        return
-    }
-    console.log("Écriture réussie")
-
-    console.log("Test de lecture...")
-    const datasetLu = await getList<Test>("Tests de l'API", new Test())
-    if (!_.isEqual(datasetLu, dataset)) {
-        console.log("ECHEC de la lecture", datasetLu, dataset)
-        return
-    }
-    console.log("Lecture réussie")
-
-    console.log("Effacement des données...")
-    const resultatEffacement = await setList<Test>("Tests de l'API", [])
-    if (!resultatEffacement) {
-        console.log("ECHEC de l'effacement")
-        return
-    }
-    console.log("Effacement réussi")
-}
-
-testGSheetAPi().then(() => console.log("Done"))

src/server/gsheets/accessors.ts

@@ -9,7 +9,7 @@ import { GoogleSpreadsheet, GoogleSpreadsheetWorksheet } from "google-spreadshee
 const CRED_PATH = path.resolve(process.cwd(), "access/gsheets.json")
 
 const REMOTE_UPDATE_DELAY = 40000
-const DELAY_AFTER_QUERY = 1000
+const DELAY_AFTER_QUERY = 2000
 
 export type ElementWithId<ElementNoId> = { id: number } & ElementNoId
 
@@ -77,7 +77,7 @@ export class Sheet<
         readonly translation: { [k in keyof Element]: string }
     ) {
         this.invertedTranslation = _.invert(this.translation)
-        this.sheetName = sheetNames[name]
+        this.sheetName = sheetNames[name] || name
         this.frenchSpecimen = _.mapValues(
             _.invert(translation),
             (englishProp: string) => (specimen as any)[englishProp]
@@ -150,20 +150,28 @@ export class Sheet<
     }
 
     dbSave(): void {
-        this.modifiedSinceSave = false
         this.saveTimestamp = +new Date()
 
-        this.dbSaveAsync()
+        try {
+            this.dbSaveAsync()
+            this.modifiedSinceSave = false
+        } catch (e) {
+            console.error("Error in dbSave: ", e)
+        }
     }
 
     dbLoad(): void {
-        this.toRunAfterLoad = []
-        this.dbLoadAsync().then(() => {
-            if (this.toRunAfterLoad) {
-                this.toRunAfterLoad.map((func) => func())
-                this.toRunAfterLoad = undefined
-            }
-        })
+        try {
+            this.toRunAfterLoad = []
+            this.dbLoadAsync().then(() => {
+                if (this.toRunAfterLoad) {
+                    this.toRunAfterLoad.map((func) => func())
+                    this.toRunAfterLoad = undefined
+                }
+            })
+        } catch (e) {
+            console.error("Error in dbLoad: ", e)
+        }
     }
 
     private async dbSaveAsync(): Promise<void> {
@@ -174,6 +182,7 @@ export class Sheet<
 
         // Load sheet into an array of objects
         const rows = await sheet.getRows()
+        await delayDBAccess()
         if (!rows[0]) {
             throw new Error(`No column types defined in sheet ${this.name}`)
         }
@@ -234,7 +243,6 @@ export class Sheet<
 
     private async dbLoadAsync(): Promise<void> {
         type StringifiedElement = Record<keyof Element, string>
-
         const sheet = await this.getGSheet()
 
         // Load sheet into an array of objects

src/utils/standardization.ts

@@ -1,5 +1,5 @@
 export function canonicalEmail(email: string): string {
-    email = email.replace(/^\s+|\s+$/g, "")
+    email = email.replace(/^\s+|\s+$/g, "").replace(/\+[^@]+/, "") // Remove +pel in pierre+pel@gmail.com
     if (/@gmail.com$/.test(email)) {
         let domain = email.replace(/^.*@/, "")
         domain = domain.replace(/^googlemail%.com$/, "gmail.com")
@@ -28,7 +28,7 @@ export function canonicalMobile(mobile: string): string {
     if (!validMobile(mobile)) {
         return ""
     }
-    let clean = trim(mobile).replace(/[-0-9. ()+]$/g, "")
+    let clean = trim(mobile).replace(/[-. ()+]/g, "")
     if (clean.length === 11) {
         clean = clean.replace(/^33/, "0")
     }
@@ -44,3 +44,30 @@ export function canonicalMobile(mobile: string): string {
 export function trim(src: string): string {
     return typeof src !== "string" ? "" : src.replace(/^\s*/, "").replace(/\s*$/, "")
 }
+
+// eslint-disable-next-line @typescript-eslint/ban-types
+export function doCleanVolunteer<
+    Element extends { firstname: string; lastname: string; email: string; mobile: string }
+>(vol: Element): void {
+    if (!validMobile(vol.mobile)) {
+        vol.mobile = ""
+    } else {
+        vol.mobile = canonicalMobile(vol.mobile)
+    }
+
+    vol.firstname = trim(vol.firstname)
+    vol.firstname = vol.firstname
+        .toLowerCase()
+        .replace(/(?<=^|[\s'-])([a-zA-Z]|[à-ú]|[À-Ú])/gi, (s) => s.toUpperCase())
+        .replace(/\b(de|d'|du|le|la)\b/gi, (s) => s.toLowerCase())
+        .replace(/\b(d'|l')/gi, (s) => s.toLowerCase())
+
+    vol.lastname = trim(vol.lastname)
+    vol.lastname = vol.lastname
+        .toLowerCase()
+        .replace(/(?<=^|[\s'-])([a-zA-Z]|[à-ú]|[À-Ú])/gi, (s) => s.toUpperCase())
+        .replace(/\b(de|d'|du|le|la)\b/gi, (s) => s.toLowerCase())
+        .replace(/\b(d'|l')/gi, (s) => s.toLowerCase())
+
+    vol.email = canonicalEmail(vol.email)
+}