fix: retry grpc calls upon service unavailable response (#1011)

* fix: retry grpc calls upon service unavailable response

* fix: retry grpc calls for session verification

.pnp.cjs

@@ -2190,10 +2190,10 @@ const RAW_RUNTIME_STATE =
       }]\
     ]],\
     ["@grpc/grpc-js", [\
-      ["npm:1.9.12", {\
-        "packageLocation": "./.yarn/cache/@grpc-grpc-js-npm-1.9.12-cb97be6754-fe13b04844.zip/node_modules/@grpc/grpc-js/",\
+      ["npm:1.9.13", {\
+        "packageLocation": "./.yarn/cache/@grpc-grpc-js-npm-1.9.13-33f9b49e10-c52150053c.zip/node_modules/@grpc/grpc-js/",\
         "packageDependencies": [\
-          ["@grpc/grpc-js", "npm:1.9.12"],\
+          ["@grpc/grpc-js", "npm:1.9.13"],\
           ["@grpc/proto-loader", "npm:0.7.10"],\
           ["@types/node", "npm:20.2.5"]\
         ],\
@@ -5532,7 +5532,7 @@ const RAW_RUNTIME_STATE =
         "packageLocation": "./packages/api-gateway/",\
         "packageDependencies": [\
           ["@standardnotes/api-gateway", "workspace:packages/api-gateway"],\
-          ["@grpc/grpc-js", "npm:1.9.12"],\
+          ["@grpc/grpc-js", "npm:1.9.13"],\
           ["@standardnotes/domain-core", "workspace:packages/domain-core"],\
           ["@standardnotes/domain-events", "workspace:packages/domain-events"],\
           ["@standardnotes/domain-events-infra", "workspace:packages/domain-events-infra"],\
@@ -5582,7 +5582,7 @@ const RAW_RUNTIME_STATE =
           ["@aws-sdk/client-sqs", "npm:3.462.0"],\
           ["@cbor-extract/cbor-extract-linux-arm64", "npm:2.1.1"],\
           ["@cbor-extract/cbor-extract-linux-x64", "npm:2.1.1"],\
-          ["@grpc/grpc-js", "npm:1.9.12"],\
+          ["@grpc/grpc-js", "npm:1.9.13"],\
           ["@simplewebauthn/server", "npm:8.1.1"],\
           ["@simplewebauthn/typescript-types", "npm:8.0.0"],\
           ["@standardnotes/api", "npm:1.26.26"],\
@@ -5809,7 +5809,7 @@ const RAW_RUNTIME_STATE =
         "packageLocation": "./packages/grpc/",\
         "packageDependencies": [\
           ["@standardnotes/grpc", "workspace:packages/grpc"],\
-          ["@grpc/grpc-js", "npm:1.9.12"],\
+          ["@grpc/grpc-js", "npm:1.9.13"],\
           ["@types/google-protobuf", "npm:3.15.10"],\
           ["google-protobuf", "npm:3.21.2"],\
           ["grpc-tools", "npm:1.12.4"],\
@@ -6082,7 +6082,7 @@ const RAW_RUNTIME_STATE =
           ["@aws-sdk/client-s3", "npm:3.462.0"],\
           ["@aws-sdk/client-sns", "npm:3.462.0"],\
           ["@aws-sdk/client-sqs", "npm:3.462.0"],\
-          ["@grpc/grpc-js", "npm:1.9.12"],\
+          ["@grpc/grpc-js", "npm:1.9.13"],\
           ["@standardnotes/api", "npm:1.26.26"],\
           ["@standardnotes/common", "workspace:packages/common"],\
           ["@standardnotes/domain-core", "workspace:packages/domain-core"],\

packages/api-gateway/package.json

@@ -31,7 +31,7 @@
     "start": "yarn node dist/bin/server.js"
   },
   "dependencies": {
-    "@grpc/grpc-js": "^1.9.12",
+    "@grpc/grpc-js": "^1.9.13",
     "@standardnotes/domain-core": "workspace:^",
     "@standardnotes/domain-events": "workspace:*",
     "@standardnotes/domain-events-infra": "workspace:*",

packages/api-gateway/src/Service/gRPC/GRPCServiceProxy.ts

@@ -8,6 +8,7 @@ import * as grpc from '@grpc/grpc-js'
 import { CrossServiceTokenCacheInterface } from '../Cache/CrossServiceTokenCacheInterface'
 import { ServiceProxyInterface } from '../Proxy/ServiceProxyInterface'
 import { GRPCSyncingServerServiceProxy } from './GRPCSyncingServerServiceProxy'
+import { Status } from '@grpc/grpc-js/build/src/constants'
 
 export class GRPCServiceProxy implements ServiceProxyInterface {
   constructor(
@@ -27,11 +28,14 @@ export class GRPCServiceProxy implements ServiceProxyInterface {
     private gRPCSyncingServerServiceProxy: GRPCSyncingServerServiceProxy,
   ) {}
 
-  async validateSession(headers: {
-    authorization: string
-    sharedVaultOwnerContext?: string
-  }): Promise<{ status: number; data: unknown; headers: { contentType: string } }> {
-    return new Promise((resolve, reject) => {
+  async validateSession(
+    headers: {
+      authorization: string
+      sharedVaultOwnerContext?: string
+    },
+    retryAttempt?: number,
+  ): Promise<{ status: number; data: unknown; headers: { contentType: string } }> {
+    const promise = new Promise((resolve, reject) => {
       try {
         const request = new AuthorizationHeader()
         request.setBearerToken(headers.authorization)
@@ -80,6 +84,32 @@ export class GRPCServiceProxy implements ServiceProxyInterface {
         return reject(error)
       }
     })
+
+    try {
+      const result = await promise
+
+      if (retryAttempt) {
+        this.logger.info(`Request to Auth Server succeeded after ${retryAttempt} retries`)
+      }
+
+      return result as { status: number; data: unknown; headers: { contentType: string } }
+    } catch (error) {
+      const requestDidNotMakeIt =
+        'code' in (error as Record<string, unknown>) && (error as Record<string, unknown>).code === Status.UNAVAILABLE
+
+      const tooManyRetryAttempts = retryAttempt && retryAttempt > 2
+      if (!tooManyRetryAttempts && requestDidNotMakeIt) {
+        await this.timer.sleep(50)
+
+        const nextRetryAttempt = retryAttempt ? retryAttempt + 1 : 1
+
+        this.logger.info(`Retrying request to Auth Server for the ${nextRetryAttempt} time`)
+
+        return this.validateSession(headers, nextRetryAttempt)
+      }
+
+      throw error
+    }
   }
 
   async callSyncingServer(
@@ -92,6 +122,21 @@ export class GRPCServiceProxy implements ServiceProxyInterface {
       payload !== undefined && typeof payload !== 'string' && 'api' in payload && payload.api === '20200115'
 
     if (requestIsUsingLatestApiVersions && endpoint === 'items/sync') {
+      await this.callSyncingServerGRPC(request, response, payload)
+
+      return
+    }
+
+    await this.callServer(this.syncingServerJsUrl, request, response, endpoint, payload)
+  }
+
+  private async callSyncingServerGRPC(
+    request: Request,
+    response: Response,
+    payload?: Record<string, unknown> | string,
+    retryAttempt?: number,
+  ): Promise<void> {
+    try {
       const result = await this.gRPCSyncingServerServiceProxy.sync(request, response, payload)
 
       response.status(result.status).send({
@@ -107,10 +152,30 @@ export class GRPCServiceProxy implements ServiceProxyInterface {
         data: result.data,
       })
 
-      return
-    }
+      if (retryAttempt) {
+        this.logger.info(`Request to Syncing Server succeeded after ${retryAttempt} retries`, {
+          userId: response.locals.user ? response.locals.user.uuid : undefined,
+        })
+      }
+    } catch (error) {
+      const requestDidNotMakeIt =
+        'code' in (error as Record<string, unknown>) && (error as Record<string, unknown>).code === Status.UNAVAILABLE
 
-    await this.callServer(this.syncingServerJsUrl, request, response, endpoint, payload)
+      const tooManyRetryAttempts = retryAttempt && retryAttempt > 2
+      if (!tooManyRetryAttempts && requestDidNotMakeIt) {
+        await this.timer.sleep(50)
+
+        const nextRetryAttempt = retryAttempt ? retryAttempt + 1 : 1
+
+        this.logger.info(`Retrying request to Syncing Server for the ${nextRetryAttempt} time`, {
+          userId: response.locals.user ? response.locals.user.uuid : undefined,
+        })
+
+        return this.callSyncingServerGRPC(request, response, payload, nextRetryAttempt)
+      }
+
+      throw error
+    }
   }
 
   async callRevisionsServer(

packages/auth/package.json

@@ -43,7 +43,7 @@
     "@aws-sdk/client-sqs": "^3.462.0",
     "@cbor-extract/cbor-extract-linux-arm64": "^2.1.1",
     "@cbor-extract/cbor-extract-linux-x64": "^2.1.1",
-    "@grpc/grpc-js": "^1.9.12",
+    "@grpc/grpc-js": "^1.9.13",
     "@simplewebauthn/server": "^8.1.1",
     "@simplewebauthn/typescript-types": "^8.0.0",
     "@standardnotes/api": "^1.26.26",

packages/grpc/package.json

@@ -27,7 +27,7 @@
     "build": "tsc --build"
   },
   "dependencies": {
-    "@grpc/grpc-js": "^1.9.12",
+    "@grpc/grpc-js": "^1.9.13",
     "google-protobuf": "^3.21.2"
   },
   "devDependencies": {

packages/syncing-server/package.json

@@ -35,7 +35,7 @@
     "@aws-sdk/client-s3": "^3.462.0",
     "@aws-sdk/client-sns": "^3.462.0",
     "@aws-sdk/client-sqs": "^3.462.0",
-    "@grpc/grpc-js": "^1.9.12",
+    "@grpc/grpc-js": "^1.9.13",
     "@standardnotes/api": "^1.26.26",
     "@standardnotes/common": "workspace:*",
     "@standardnotes/domain-core": "workspace:^",

yarn.lock

@@ -1583,13 +1583,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@grpc/grpc-js@npm:^1.9.12":
-  version: 1.9.12
-  resolution: "@grpc/grpc-js@npm:1.9.12"
+"@grpc/grpc-js@npm:^1.9.13":
+  version: 1.9.13
+  resolution: "@grpc/grpc-js@npm:1.9.13"
   dependencies:
     "@grpc/proto-loader": "npm:^0.7.8"
     "@types/node": "npm:>=12.12.47"
-  checksum: fe13b04844b525ad860521589e2d640bb8cfeea46e3cb8e4eab537e0a4fcb04a033083c25d5c3cd4e061a6471c933f6f12e81dcc626acdcf68435e6e4a833a06
+  checksum: c52150053ca3911bf9ec5012265aa754627aba9c60577ef07c594c5c22896e939ec0f656cc130a54a8651ea0ae23f385a4a48868fc71ff56dff54eeaec8b6912
   languageName: node
   linkType: hard
 
@@ -4084,7 +4084,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@standardnotes/api-gateway@workspace:packages/api-gateway"
   dependencies:
-    "@grpc/grpc-js": "npm:^1.9.12"
+    "@grpc/grpc-js": "npm:^1.9.13"
     "@standardnotes/domain-core": "workspace:^"
     "@standardnotes/domain-events": "workspace:*"
     "@standardnotes/domain-events-infra": "workspace:*"
@@ -4147,7 +4147,7 @@ __metadata:
     "@aws-sdk/client-sqs": "npm:^3.462.0"
     "@cbor-extract/cbor-extract-linux-arm64": "npm:^2.1.1"
     "@cbor-extract/cbor-extract-linux-x64": "npm:^2.1.1"
-    "@grpc/grpc-js": "npm:^1.9.12"
+    "@grpc/grpc-js": "npm:^1.9.13"
     "@simplewebauthn/server": "npm:^8.1.1"
     "@simplewebauthn/typescript-types": "npm:^8.0.0"
     "@standardnotes/api": "npm:^1.26.26"
@@ -4361,7 +4361,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@standardnotes/grpc@workspace:packages/grpc"
   dependencies:
-    "@grpc/grpc-js": "npm:^1.9.12"
+    "@grpc/grpc-js": "npm:^1.9.13"
     "@types/google-protobuf": "npm:^3"
     google-protobuf: "npm:^3.21.2"
     grpc-tools: "npm:^1.12.4"
@@ -4618,7 +4618,7 @@ __metadata:
     "@aws-sdk/client-s3": "npm:^3.462.0"
     "@aws-sdk/client-sns": "npm:^3.462.0"
     "@aws-sdk/client-sqs": "npm:^3.462.0"
-    "@grpc/grpc-js": "npm:^1.9.12"
+    "@grpc/grpc-js": "npm:^1.9.13"
     "@standardnotes/api": "npm:^1.26.26"
     "@standardnotes/common": "workspace:*"
     "@standardnotes/domain-core": "workspace:^"