PeL/force-orange-2022
PeL/force-orange-2022
1
1
Fork 0
mirror of https://github.com/Paris-est-Ludique/intranet.git synced 2025-06-08 08:34:20 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix gsheet api request saturation caused by an infinite delayed loop

Browse Source
This commit is contained in:
pikiou 2022-02-22 02:10:16 +01:00
parent d8e7bb9f70
commit 968ec3b430
3 changed files with 61 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/server/gsheets/accessors.ts
View File

@ -11,8 +11,9 @@ export { SheetNames } from "./localDb"
const CRED_PATH = path.resolve(process.cwd(), "access/gsheets.json")
const REMOTE_UPDATE_DELAY = 80000
const REMOTE_UPDATE_DELAY = 40000
const DELAY_BETWEEN_ATTEMPTS = 10000
const DELAY_BETWEEN_FIRST_LOAD = 1500
let creds: string | undefined | null
@ -44,7 +45,7 @@ export async function checkGSheetsAccess(): Promise<void> {
console.error(`Google Sheets: no creds found, loading local database instead`)
}
}
export function getSheet<
export async function getSheet<
// eslint-disable-next-line @typescript-eslint/ban-types
ElementNoId extends object,
Element extends ElementNoId & ElementWithId<ElementNoId>
@ -52,18 +53,17 @@ export function getSheet<
sheetName: keyof SheetNames,
specimen: Element,
translation: { [k in keyof Element]: string }
): Sheet<ElementNoId, Element> {
): Promise<Sheet<ElementNoId, Element>> {
let sheet: Sheet<ElementNoId, Element>
if (!sheetList[sheetName]) {
sheetList[sheetName] = new Sheet<ElementNoId, Element>(sheetName, specimen, translation)
sheet = new Sheet<ElementNoId, Element>(sheetName, specimen, translation)
await sheet.waitForFirstLoad()
sheetList[sheetName] = sheet
setInterval(() => sheet.dbUpdate(), REMOTE_UPDATE_DELAY)
} else {
sheet = sheetList[sheetName] as Sheet<ElementNoId, Element>
}
const sheet = sheetList[sheetName] as Sheet<ElementNoId, Element>
setTimeout(
() => setInterval(() => sheet.dbUpdate(), REMOTE_UPDATE_DELAY),
1000 * Object.values(sheetList).length
)
return sheet
}
@ -100,8 +100,6 @@ export class Sheet<
_.invert(translation),
(englishProp: string) => (specimen as any)[englishProp]
) as Element
setTimeout(() => this.dbFirstLoad(), 100 * Object.values(sheetList).length)
}
async getList(): Promise<Element[] | undefined> {
@ -154,6 +152,15 @@ export class Sheet<
}
}
async waitForFirstLoad(): Promise<void> {
setTimeout(
() => this.dbFirstLoad(),
DELAY_BETWEEN_FIRST_LOAD * Object.values(sheetList).length
)
await this.waitForLoad()
}
private async waitForLoad(): Promise<void> {
return new Promise((resolve, _reject) => {
this.addToRunAfterLoad(() => resolve(undefined))
@ -194,10 +201,9 @@ export class Sheet<
async dbLoad(): Promise<void> {
try {
if (await hasGSheetsAccess()) {
this.dbLoadAsync().then(() => this.doRunAfterLoad())
} else {
this.doRunAfterLoad()
await this.dbLoadAsync()
}
this.doRunAfterLoad()
} catch (e) {
console.error("Error in dbLoad: ", e)
}
@ -214,6 +220,7 @@ export class Sheet<
if (!(await hasGSheetsAccess())) {
await this.loadLocalDb()
} else if (this.toRunAfterLoad && __DEV__) {
// Save once
this.toRunAfterLoad.push(() => this.saveLocalDb())
}
@ -288,7 +295,7 @@ export class Sheet<
private async dbLoadAsync(): Promise<void> {
type StringifiedElement = Record<keyof Element, string>
const sheet = await this.getGSheet()
const sheet = await this.getGSheet(20)
if (!sheet) {
return
@ -326,7 +333,7 @@ export class Sheet<
})
}
private async getGSheet(): Promise<GoogleSpreadsheetWorksheet | null> {
private async getGSheet(attempts = 3): Promise<GoogleSpreadsheetWorksheet | null> {
return tryNTimes(
async () => {
if (creds === undefined) {
@ -347,7 +354,7 @@ export class Sheet<
return doc.sheetsByTitle[this.sheetName]
},
() => null,
20,
attempts,
DELAY_BETWEEN_ATTEMPTS / 5
)
}
@ -590,7 +597,7 @@ function parseDate(value: string): Date {
async function tryNTimes<T>(
func: () => Promise<T> | T,
failFunc?: () => Promise<T> | T,
repeatCount = 5,
repeatCount = 2,
delayBetweenAttempts = DELAY_BETWEEN_ATTEMPTS
): Promise<T> {
try {
@ -601,7 +608,7 @@ async function tryNTimes<T>(
await new Promise<void>((resolve) => {
setTimeout(() => resolve(), delayBetweenAttempts)
})
if (repeatCount === 1) {
if (repeatCount <= 1) {
console.error(`No more attempts left every ${delayBetweenAttempts}`)
if (failFunc) {
return failFunc()
@ -614,7 +621,7 @@ async function tryNTimes<T>(
async function tryNTimesVoidReturn(
func: () => Promise<void> | void,
repeatCount = 5,
repeatCount = 2,
delayBetweenAttempts = DELAY_BETWEEN_ATTEMPTS
): Promise<void> {
return tryNTimes(func, () => undefined, repeatCount, delayBetweenAttempts)

38
src/server/gsheets/expressAccessors.ts
View File

@ -9,14 +9,33 @@ export default class ExpressAccessors<
ElementNoId extends object,
Element extends ElementWithId<ElementNoId>
> {
sheet: Sheet<ElementNoId, Element>
sheet?: Sheet<ElementNoId, Element>
runAfterLoad: ((value: Sheet<ElementNoId, Element>) => void)[] = []
isLoaded = false
constructor(
readonly sheetName: keyof SheetNames,
readonly specimen: Element,
readonly translation: { [k in keyof Element]: string }
) {
this.sheet = getSheet<ElementNoId, Element>(sheetName, specimen, translation)
getSheet<ElementNoId, Element>(this.sheetName, this.specimen, this.translation).then(
(sheet) => {
this.sheet = sheet
this.isLoaded = true
this.runAfterLoad.map((f) => f(sheet))
}
)
}
async getSheet(): Promise<Sheet<ElementNoId, Element>> {
if (!this.isLoaded) {
await new Promise((resolve) => {
this.runAfterLoad.push(resolve)
})
}
return this.sheet as Sheet<ElementNoId, Element>
}
listGet() {
@ -26,7 +45,7 @@ export default class ExpressAccessors<
_next: NextFunction
): Promise<void> => {
try {
const elements = await this.sheet.getList()
const elements = await (await this.getSheet()).getList()
if (elements) {
response.status(200).json(elements)
}
@ -42,7 +61,8 @@ export default class ExpressAccessors<
) {
return async (request: Request, response: Response, _next: NextFunction): Promise<void> => {
try {
const list = (await this.sheet.getList()) || []
const sheet = await this.getSheet()
const list = (await sheet.getList()) || []
let toCaller: any
if (!custom) {
const id = parseInt(request.query.id as string, 10) || -1
@ -65,7 +85,8 @@ export default class ExpressAccessors<
add() {
return async (request: Request, response: Response, _next: NextFunction): Promise<void> => {
try {
const element: Element = await this.sheet.add(request.body)
const sheet = await this.getSheet()
const element: Element = await sheet.add(request.body)
if (element) {
response.status(200).json(element)
}
@ -85,15 +106,16 @@ export default class ExpressAccessors<
) {
return async (request: Request, response: Response, _next: NextFunction): Promise<void> => {
try {
const sheet = await this.getSheet()
if (!custom) {
await this.sheet.set(request.body)
await sheet.set(request.body)
response.status(200)
} else {
const memberId = response?.locals?.jwt?.id || -1
const list = (await this.sheet.getList()) || []
const list = (await sheet.getList()) || []
const { toDatabase, toCaller } = await custom(list, request.body, memberId)
if (toDatabase !== undefined) {
await this.sheet.set(toDatabase)
await sheet.set(toDatabase)
}
if (toCaller !== undefined) {
response.status(200).json(toCaller)

4
src/server/notifications.ts
View File

@ -80,7 +80,7 @@ export function notificationMain(): void {
}
async function notifyAboutAnnouncement(): Promise<void> {
const announcementSheet = getSheet<AnnouncementWithoutId, Announcement>(
const announcementSheet = await getSheet<AnnouncementWithoutId, Announcement>(
"Announcements",
new Announcement(),
translationAnnouncement
@ -96,7 +96,7 @@ async function notifyAboutAnnouncement(): Promise<void> {
return
}
const volunteerSheet = getSheet<VolunteerWithoutId, Volunteer>(
const volunteerSheet = await getSheet<VolunteerWithoutId, Volunteer>(
"Volunteers",
new Volunteer(),
translationVolunteer