Retry replication unless the error is a known non-retry error

Flip the strategy to retry by default

Author: matt-aitken

apps/webapp/app/services/runsReplicationService.server.ts

@@ -515,11 +515,11 @@ export class RunsReplicationService {
       } catch (error) {
         lastError = error instanceof Error ? error : new Error(String(error));
 
-        // Check if this is a retryable connection error
-        if (this.#isRetryableConnectionError(lastError)) {
-          const delay = this.#calculateConnectionRetryDelay(attempt);
+        // Check if this is a retryable error
+        if (this.#isRetryableError(lastError)) {
+          const delay = this.#calculateRetryDelay(attempt);
 
-          this.logger.warn(`Retrying RunReplication insert due to connection error`, {
+          this.logger.warn(`Retrying RunReplication insert due to error`, {
             operationName,
             flushId,
             attempt,
@@ -538,26 +538,37 @@ export class RunsReplicationService {
     return [lastError, null];
   }
 
-  // New method to check if an error is a retryable connection error
-  #isRetryableConnectionError(error: Error): boolean {
+  // Retry all errors except known permanent ones
+  #isRetryableError(error: Error): boolean {
     const errorMessage = error.message.toLowerCase();
-    const retryableConnectionPatterns = [
-      "socket hang up",
-      "econnreset",
-      "connection reset",
-      "connection refused",
-      "connection timeout",
-      "network error",
-      "read econnreset",
-      "write econnreset",
-      "timeout",
+
+    // Permanent errors that should NOT be retried
+    const permanentErrorPatterns = [
+      "authentication failed",
+      "permission denied",
+      "invalid credentials",
+      "table not found",
+      "database not found",
+      "column not found",
+      "schema mismatch",
+      "invalid query",
+      "syntax error",
+      "type error",
+      "constraint violation",
+      "duplicate key",
+      "foreign key violation",
     ];
 
-    return retryableConnectionPatterns.some((pattern) => errorMessage.includes(pattern));
+    // If it's a known permanent error, don't retry
+    if (permanentErrorPatterns.some((pattern) => errorMessage.includes(pattern))) {
+      return false;
+    }
+
+    // Retry everything else
+    return true;
   }
 
-  // New method to calculate retry delay for connection errors
-  #calculateConnectionRetryDelay(attempt: number): number {
+  #calculateRetryDelay(attempt: number): number {
     // Exponential backoff: baseDelay, baseDelay*2, baseDelay*4, etc.
     const delay = Math.min(
       this._insertBaseDelayMs * Math.pow(2, attempt - 1),