PeL/force-orange-2022
PeL/force-orange-2022
1
1
Fork 0
mirror of https://github.com/Paris-est-Ludique/intranet.git synced 2025-06-09 17:14:21 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add crash safety when accessing Google Sheet API by trying multiple times

Browse Source
This commit is contained in:
pikiou 2022-02-03 17:16:30 +01:00
parent 3264336aab
commit bd8a74cd17
3 changed files with 141 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -14,6 +14,9 @@ public/*
# Access # Access
access/* access/*
# Gazettes
gazettes/*
# Misc # Misc
.DS_Store .DS_Store
*.log *.log

4
.prettierignore
View File

@ -7,4 +7,6 @@ public/*
# Misc # Misc
*.log *.log
node_modules/*
access/*
gazettes/*

228
src/server/gsheets/accessors.ts
View File

@ -230,65 +230,67 @@ export class Sheet<
return return
} }
// Load sheet into an array of objects await tryNTimesVoidReturn(async () => {
const rows = await sheet.getRows() // Load sheet into an array of objects
await delayDBAccess() const rows = await sheet.getRows()
if (!rows[0]) { await delayDBAccess()
throw new Error(`No column types defined in sheet ${this.name}`) if (!rows[0]) {
} throw new Error(`No column types defined in sheet ${this.name}`)
// eslint-disable-next-line @typescript-eslint/ban-types }
const elements = this._state as Element[] // eslint-disable-next-line @typescript-eslint/ban-types
this._type = _.pick(rows[0], Object.values(this.translation)) as Record< const elements = this._state as Element[]
keyof Element, this._type = _.pick(rows[0], Object.values(this.translation)) as Record<
string keyof Element,
> string
>
// Update received rows // Update received rows
let rowid = 1 let rowid = 1
// eslint-disable-next-line no-restricted-syntax // eslint-disable-next-line no-restricted-syntax
for (const element of elements) { for (const element of elements) {
const row = rows[rowid] const row = rows[rowid]
const frenchElement = _.mapValues( const frenchElement = _.mapValues(
this.invertedTranslation, this.invertedTranslation,
(englishProp: string) => (element as any)[englishProp] (englishProp: string) => (element as any)[englishProp]
) as Element ) as Element
const stringifiedRow = this.stringifyElement(frenchElement, this._type) const stringifiedRow = this.stringifyElement(frenchElement, this._type)
if (!row) { if (!row) {
// eslint-disable-next-line no-await-in-loop
await sheet.addRow(stringifiedRow)
// eslint-disable-next-line no-await-in-loop
await delayDBAccess()
} else {
const keys = Object.keys(stringifiedRow)
const sameCells = _.every(keys, (key: keyof Element) => {
const rawVal = row[key as string]
const val: string = rawVal === undefined ? "" : rawVal
return val === stringifiedRow[key]
})
if (!sameCells) {
keys.forEach((key) => {
row[key] = stringifiedRow[key as keyof Element]
})
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
await row.save() await sheet.addRow(stringifiedRow)
// eslint-disable-next-line no-await-in-loop
await delayDBAccess()
} else {
const keys = Object.keys(stringifiedRow)
const sameCells = _.every(keys, (key: keyof Element) => {
const rawVal = row[key as string]
const val: string = rawVal === undefined ? "" : rawVal
return val === stringifiedRow[key]
})
if (!sameCells) {
keys.forEach((key) => {
row[key] = stringifiedRow[key as keyof Element]
})
// eslint-disable-next-line no-await-in-loop
await row.save()
// eslint-disable-next-line no-await-in-loop
await delayDBAccess()
}
}
rowid += 1
}
// Delete all following rows
for (let rowToDelete = sheet.rowCount - 1; rowToDelete >= rowid; rowToDelete -= 1) {
if (rows[rowToDelete]) {
// eslint-disable-next-line no-await-in-loop
await rows[rowToDelete].delete()
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
await delayDBAccess() await delayDBAccess()
} }
} }
})
rowid += 1
}
// Delete all following rows
for (let rowToDelete = sheet.rowCount - 1; rowToDelete >= rowid; rowToDelete -= 1) {
if (rows[rowToDelete]) {
// eslint-disable-next-line no-await-in-loop
await rows[rowToDelete].delete()
// eslint-disable-next-line no-await-in-loop
await delayDBAccess()
}
}
} }
private async dbLoadAsync(): Promise<void> { private async dbLoadAsync(): Promise<void> {
@ -299,54 +301,61 @@ export class Sheet<
return return
} }
// Load sheet into an array of objects await tryNTimesVoidReturn(async () => {
const rows = (await sheet.getRows()) as StringifiedElement[] // Load sheet into an array of objects
await delayDBAccess() const rows = (await sheet.getRows()) as StringifiedElement[]
const elements: Element[] = [] await delayDBAccess()
if (!rows[0]) { const elements: Element[] = []
throw new Error(`No column types defined in sheet ${this.name}`) if (!rows[0]) {
} throw new Error(`No column types defined in sheet ${this.name}`)
const typeList = _.pick(rows[0], Object.values(this.translation)) as Record< }
keyof Element, const typeList = _.pick(rows[0], Object.values(this.translation)) as Record<
string
>
this._type = typeList
rows.shift()
rows.forEach((row) => {
const stringifiedElement = _.pick(row, Object.values(this.translation)) as Record<
keyof Element, keyof Element,
string string
> >
const frenchData: any = this.parseElement(stringifiedElement, typeList) this._type = typeList
if (frenchData !== undefined) { rows.shift()
const englishElement = _.mapValues( rows.forEach((row) => {
this.translation, const stringifiedElement = _.pick(row, Object.values(this.translation)) as Record<
(frenchProp: string) => frenchData[frenchProp] keyof Element,
) as Element string
elements.push(englishElement) >
} const frenchData: any = this.parseElement(stringifiedElement, typeList)
}) if (frenchData !== undefined) {
const englishElement = _.mapValues(
this.translation,
(frenchProp: string) => frenchData[frenchProp]
) as Element
elements.push(englishElement)
}
})
this._state = elements this._state = elements
})
} }
private async getGSheet(): Promise<GoogleSpreadsheetWorksheet | null> { private async getGSheet(): Promise<GoogleSpreadsheetWorksheet | null> {
if (creds === undefined) { return tryNTimes(
if (await hasGSheetsAccess()) { async () => {
const credsBuffer = await fs.readFile(CRED_PATH) if (creds === undefined) {
creds = credsBuffer?.toString() || null if (await hasGSheetsAccess()) {
} else { const credsBuffer = await fs.readFile(CRED_PATH)
creds = null creds = credsBuffer?.toString() || null
} } else {
} creds = null
if (creds === null) { }
return null }
} if (creds === null) {
// Authentication return null
const doc = new GoogleSpreadsheet("1pMMKcYx6NXLOqNn6pLHJTPMTOLRYZmSNg2QQcAu7-Pw") }
await doc.useServiceAccountAuth(JSON.parse(creds)) // Authentication
await doc.loadInfo() const doc = new GoogleSpreadsheet("1pMMKcYx6NXLOqNn6pLHJTPMTOLRYZmSNg2QQcAu7-Pw")
return doc.sheetsByTitle[this.sheetName] await doc.useServiceAccountAuth(JSON.parse(creds))
await doc.loadInfo()
return doc.sheetsByTitle[this.sheetName]
},
() => null
)
} }
private parseElement( private parseElement(
@ -579,3 +588,36 @@ function parseDate(value: string): Date {
async function delayDBAccess(): Promise<void> { async function delayDBAccess(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, DELAY_AFTER_QUERY)) return new Promise((resolve) => setTimeout(resolve, DELAY_AFTER_QUERY))
} }
async function tryNTimes<T>(
func: () => Promise<T> | T,
failFunc?: () => Promise<T> | T,
repeatCount = 5,
delayBetweenAttempts = 2000
): Promise<T> {
try {
return await func()
} catch (e: any) {
console.error(e?.error || e?.message || e)
console.error(`${repeatCount} attemps left every ${delayBetweenAttempts}`)
await new Promise<void>((resolve) => {
setTimeout(() => resolve(), delayBetweenAttempts)
})
if (repeatCount === 1) {
console.error(`No more attempts left every ${delayBetweenAttempts}`)
if (failFunc) {
return failFunc()
}
throw Error(e)
}
return tryNTimes(func, failFunc, repeatCount - 1, delayBetweenAttempts)
}
}
async function tryNTimesVoidReturn(
func: () => Promise<void> | void,
repeatCount = 5,
delayBetweenAttempts = 2000
): Promise<void> {
return tryNTimes(func, () => undefined, repeatCount, delayBetweenAttempts)
}