Add crash safety when accessing Google Sheet API by trying multiple times

.gitignore

@@ -14,6 +14,9 @@ public/*
 # Access
 access/*
 
+# Gazettes
+gazettes/*
+
 # Misc
 .DS_Store
 *.log

.prettierignore

@@ -7,4 +7,6 @@ public/*
 
 # Misc
 *.log
-
+node_modules/*
+access/*
+gazettes/*

src/server/gsheets/accessors.ts

@@ -230,6 +230,7 @@ export class Sheet<
             return
         }
 
+        await tryNTimesVoidReturn(async () => {
             // Load sheet into an array of objects
             const rows = await sheet.getRows()
             await delayDBAccess()
@@ -289,6 +290,7 @@ export class Sheet<
                     await delayDBAccess()
                 }
             }
+        })
     }
 
     private async dbLoadAsync(): Promise<void> {
@@ -299,6 +301,7 @@ export class Sheet<
             return
         }
 
+        await tryNTimesVoidReturn(async () => {
             // Load sheet into an array of objects
             const rows = (await sheet.getRows()) as StringifiedElement[]
             await delayDBAccess()
@@ -328,9 +331,12 @@ export class Sheet<
             })
 
             this._state = elements
+        })
     }
 
     private async getGSheet(): Promise<GoogleSpreadsheetWorksheet | null> {
+        return tryNTimes(
+            async () => {
                 if (creds === undefined) {
                     if (await hasGSheetsAccess()) {
                         const credsBuffer = await fs.readFile(CRED_PATH)
@@ -347,6 +353,9 @@ export class Sheet<
                 await doc.useServiceAccountAuth(JSON.parse(creds))
                 await doc.loadInfo()
                 return doc.sheetsByTitle[this.sheetName]
+            },
+            () => null
+        )
     }
 
     private parseElement(
@@ -579,3 +588,36 @@ function parseDate(value: string): Date {
 async function delayDBAccess(): Promise<void> {
     return new Promise((resolve) => setTimeout(resolve, DELAY_AFTER_QUERY))
 }
+
+async function tryNTimes<T>(
+    func: () => Promise<T> | T,
+    failFunc?: () => Promise<T> | T,
+    repeatCount = 5,
+    delayBetweenAttempts = 2000
+): Promise<T> {
+    try {
+        return await func()
+    } catch (e: any) {
+        console.error(e?.error || e?.message || e)
+        console.error(`${repeatCount} attemps left every ${delayBetweenAttempts}`)
+        await new Promise<void>((resolve) => {
+            setTimeout(() => resolve(), delayBetweenAttempts)
+        })
+        if (repeatCount === 1) {
+            console.error(`No more attempts left every ${delayBetweenAttempts}`)
+            if (failFunc) {
+                return failFunc()
+            }
+            throw Error(e)
+        }
+        return tryNTimes(func, failFunc, repeatCount - 1, delayBetweenAttempts)
+    }
+}
+
+async function tryNTimesVoidReturn(
+    func: () => Promise<void> | void,
+    repeatCount = 5,
+    delayBetweenAttempts = 2000
+): Promise<void> {
+    return tryNTimes(func, () => undefined, repeatCount, delayBetweenAttempts)
+}