Performance & Optimization - 2/2

// The caching nightmare that destroys performance
class ProductService {
  private cache = new Map(); // In-memory cache - RED FLAG #1
  private redisCache: RedisClient;
  private fileCache = new FileSystem(); // Three caching layers - RED FLAG #2

  async getProduct(productId: string) {
    // Multiple cache layers without coordination - RED FLAG #3

    // Check memory cache
    if (this.cache.has(productId)) {
      return this.cache.get(productId);
    }

    // Check Redis cache
    const redisData = await this.redisCache.get(productId);
    if (redisData) {
      const product = JSON.parse(redisData);
      this.cache.set(productId, product); // Cache duplication - RED FLAG #4
      return product;
    }

    // Check file cache
    try {
      const fileData = await this.fileCache.readFile(`cache/${productId}.json`);
      const product = JSON.parse(fileData);

      // Store in all cache layers - RED FLAG #5
      this.cache.set(productId, product);
      await this.redisCache.setex(productId, 3600, JSON.stringify(product));

      return product;
    } catch (error) {
      // File doesn't exist, continue to database
    }

    // Finally load from database
    const product = await this.database.getProduct(productId);

    // Cache invalidation nightmare - RED FLAG #6
    // When do we clear these caches? How do we keep them in sync?
    this.cache.set(productId, product);
    await this.redisCache.setex(productId, 3600, JSON.stringify(product));
    await this.fileCache.writeFile(
      `cache/${productId}.json`,
      JSON.stringify(product)
    );

    return product;
  }

  async updateProduct(productId: string, updates: any) {
    const product = await this.database.updateProduct(productId, updates);

    // Partial cache invalidation - things will go wrong - RED FLAG #7
    this.cache.delete(productId);
    await this.redisCache.del(productId);

    // But what about related caches?
    // - Category cache still has old data
    // - Search results cache still has old data
    // - User's "recently viewed" cache still has old data
    // - File cache still exists and will be used next time

    return product;
  }

  async searchProducts(query: string) {
    const cacheKey = `search:${query}`;

    // Cache key collision waiting to happen - RED FLAG #8
    const cached = await this.redisCache.get(cacheKey);
    if (cached) {
      return JSON.parse(cached);
    }

    const results = await this.database.searchProducts(query);

    // Fixed TTL regardless of data volatility - RED FLAG #9
    await this.redisCache.setex(cacheKey, 300, JSON.stringify(results));

    return results;

    // Problems this creates:
    // - Memory usage grows unbounded (Map cache never clears)
    // - Cache stampede when TTL expires on popular queries
    // - No cache warming for critical data
    // - No cache compression for large objects
    // - No cache analytics to optimize hit rates
  }
}

// Sophisticated caching system with proper invalidation
export interface CacheEntry<T> {
  data: T;
  timestamp: number;
  ttl: number;
  hitCount: number;
  tags: string[];
}

export interface CacheMetrics {
  hits: number;
  misses: number;
  evictions: number;
  memoryUsage: number;
  averageHitTime: number;
  averageMissTime: number;
}

export class IntelligentCacheManager {
  private l1Cache: LRUCache<string, CacheEntry<any>>; // L1: In-memory
  private l2Cache: RedisCluster; // L2: Redis cluster
  private l3Cache: DatabaseCache; // L3: Database query cache
  private metrics: CacheMetrics;
  private invalidationQueue: Queue<InvalidationEvent>;

  constructor(config: CacheConfig) {
    // L1 cache: Small, fast, process-local
    this.l1Cache = new LRUCache({
      max: config.l1MaxEntries || 10000,
      maxAge: config.l1MaxAge || 60000, // 1 minute
      updateAgeOnGet: true,
      dispose: (key, entry) => this.recordEviction("l1", key, entry),
    });

    // L2 cache: Larger, shared across instances
    this.l2Cache = new Redis.Cluster(config.redisNodes, {
      retryDelayOnFailover: 100,
      enableReadyCheck: true,
      maxRetriesPerRequest: 3,
      compression: "gzip", // Compress large values
    });

    this.initializeMetrics();
    this.startInvalidationWorker();
  }

  async get<T>(key: string, tags: string[] = []): Promise<T | null> {
    const startTime = Date.now();

    try {
      // L1 cache check
      const l1Result = this.l1Cache.get(key);
      if (l1Result && !this.isExpired(l1Result)) {
        this.recordHit("l1", Date.now() - startTime);
        l1Result.hitCount++;
        return l1Result.data;
      }

      // L2 cache check
      const l2Result = await this.getFromL2(key);
      if (l2Result) {
        this.recordHit("l2", Date.now() - startTime);

        // Populate L1 cache
        this.l1Cache.set(key, l2Result);
        return l2Result.data;
      }

      this.recordMiss(Date.now() - startTime);
      return null;
    } catch (error) {
      this.recordError("cache_get", error);
      return null;
    }
  }

  async set<T>(
    key: string,
    data: T,
    options: CacheSetOptions = {}
  ): Promise<void> {
    const ttl = options.ttl || this.calculateDynamicTTL(key, data);
    const tags = options.tags || [];

    const entry: CacheEntry<T> = {
      data,
      timestamp: Date.now(),
      ttl,
      hitCount: 0,
      tags,
    };

    try {
      // Store in both L1 and L2
      await Promise.all([this.setInL1(key, entry), this.setInL2(key, entry)]);

      // Track tags for invalidation
      if (tags.length > 0) {
        await this.trackTaggedEntry(key, tags);
      }
    } catch (error) {
      this.recordError("cache_set", error);
      throw error;
    }
  }

  async invalidateByTags(tags: string[]): Promise<void> {
    const invalidationEvent: InvalidationEvent = {
      type: "tags",
      tags,
      timestamp: Date.now(),
    };

    // Queue for processing to avoid blocking
    await this.invalidationQueue.add("invalidate", invalidationEvent);
  }

  async invalidatePattern(pattern: string): Promise<void> {
    const invalidationEvent: InvalidationEvent = {
      type: "pattern",
      pattern,
      timestamp: Date.now(),
    };

    await this.invalidationQueue.add("invalidate", invalidationEvent);
  }

  private async setInL1<T>(key: string, entry: CacheEntry<T>): Promise<void> {
    // Intelligent L1 caching based on access patterns
    if (this.shouldCacheInL1(key, entry)) {
      this.l1Cache.set(key, entry);
    }
  }

  private async setInL2<T>(key: string, entry: CacheEntry<T>): Promise<void> {
    const serialized = this.serialize(entry);
    const compressed = await this.compress(serialized);

    await this.l2Cache.setex(key, Math.floor(entry.ttl / 1000), compressed);
  }

  private async getFromL2(key: string): Promise<CacheEntry<any> | null> {
    try {
      const compressed = await this.l2Cache.get(key);
      if (!compressed) return null;

      const serialized = await this.decompress(compressed);
      const entry = this.deserialize(serialized);

      return this.isExpired(entry) ? null : entry;
    } catch (error) {
      this.recordError("l2_get", error);
      return null;
    }
  }

  private shouldCacheInL1<T>(key: string, entry: CacheEntry<T>): boolean {
    // Cache small, frequently accessed items in L1
    const size = this.estimateSize(entry.data);
    const isFrequentlyAccessed = entry.hitCount > 10;
    const isSmall = size < 1024; // 1KB threshold

    return isSmall && (isFrequentlyAccessed || this.isHotKey(key));
  }

  private calculateDynamicTTL<T>(key: string, data: T): number {
    // Dynamic TTL based on data characteristics
    if (key.startsWith("user:")) {
      return 300000; // 5 minutes for user data
    } else if (key.startsWith("product:")) {
      return 3600000; // 1 hour for product data
    } else if (key.startsWith("search:")) {
      return 600000; // 10 minutes for search results
    } else if (key.startsWith("static:")) {
      return 86400000; // 24 hours for static content
    }

    return 1800000; // 30 minutes default
  }

  private async processInvalidation(
    job: Job<InvalidationEvent>
  ): Promise<void> {
    const event = job.data;

    try {
      if (event.type === "tags") {
        await this.invalidateByTagsImpl(event.tags!);
      } else if (event.type === "pattern") {
        await this.invalidateByPatternImpl(event.pattern!);
      }
    } catch (error) {
      this.recordError("invalidation", error);
      throw error;
    }
  }

  private async invalidateByTagsImpl(tags: string[]): Promise<void> {
    for (const tag of tags) {
      const keys = await this.getKeysForTag(tag);

      // Batch invalidation for efficiency
      await this.batchInvalidate(keys);
    }
  }

  private async batchInvalidate(keys: string[]): Promise<void> {
    const batchSize = 100;
    const batches = this.chunk(keys, batchSize);

    await Promise.all(batches.map((batch) => this.invalidateBatch(batch)));
  }

  private async invalidateBatch(keys: string[]): Promise<void> {
    // L1 cache invalidation
    keys.forEach((key) => this.l1Cache.del(key));

    // L2 cache invalidation
    if (keys.length === 1) {
      await this.l2Cache.del(keys[0]);
    } else {
      await this.l2Cache.del(...keys);
    }
  }

  async warmCache(warmupConfig: CacheWarmupConfig): Promise<void> {
    const keys = await this.getCriticalKeys(warmupConfig);

    const warmupPromises = keys.map(async (key) => {
      try {
        const data = await warmupConfig.dataLoader(key);
        await this.set(key, data, {
          ttl: warmupConfig.ttl,
          tags: warmupConfig.tags,
        });
      } catch (error) {
        this.recordError("cache_warmup", error);
      }
    });

    await Promise.all(warmupPromises);
  }

  getMetrics(): CacheMetrics {
    return {
      ...this.metrics,
      l1Size: this.l1Cache.length,
      l1MemoryUsage: this.estimateL1MemoryUsage(),
      hitRatio: this.metrics.hits / (this.metrics.hits + this.metrics.misses),
    };
  }

  private isExpired(entry: CacheEntry<any>): boolean {
    return Date.now() - entry.timestamp > entry.ttl;
  }

  private isHotKey(key: string): boolean {
    // Implement hot key detection based on access patterns
    return this.hotKeys.has(key);
  }

  private serialize<T>(entry: CacheEntry<T>): string {
    return JSON.stringify(entry);
  }

  private deserialize<T>(data: string): CacheEntry<T> {
    return JSON.parse(data);
  }

  private async compress(data: string): Promise<Buffer> {
    if (data.length < 1024) return Buffer.from(data); // Don't compress small data
    return zlib.gzipSync(Buffer.from(data));
  }

  private async decompress(data: Buffer): Promise<string> {
    try {
      return zlib.gunzipSync(data).toString();
    } catch {
      return data.toString(); // Fallback for uncompressed data
    }
  }
}

// Intelligent product service using sophisticated caching
export class OptimizedProductService {
  constructor(
    private cache: IntelligentCacheManager,
    private database: ProductDatabase,
    private metrics: IPerformanceMetrics
  ) {}

  async getProduct(productId: string): Promise<Product | null> {
    const cacheKey = `product:${productId}`;
    const tags = [
      "product",
      `category:${await this.getProductCategory(productId)}`,
    ];

    // Try cache first
    const cached = await this.cache.get<Product>(cacheKey, tags);
    if (cached) {
      this.metrics.incrementCounter("product_cache_hit");
      return cached;
    }

    this.metrics.incrementCounter("product_cache_miss");

    // Load from database
    const product = await this.database.getProduct(productId);
    if (!product) return null;

    // Cache with intelligent TTL and tags
    await this.cache.set(cacheKey, product, {
      ttl: this.calculateProductTTL(product),
      tags: [
        "product",
        `category:${product.categoryId}`,
        `brand:${product.brandId}`,
      ],
    });

    return product;
  }

  async updateProduct(
    productId: string,
    updates: Partial<Product>
  ): Promise<Product> {
    const product = await this.database.updateProduct(productId, updates);

    // Smart cache invalidation
    const tags = [
      "product",
      `category:${product.categoryId}`,
      `brand:${product.brandId}`,
      "search:*", // Invalidate all search caches
      "recommendations:*", // Invalidate recommendations
    ];

    await this.cache.invalidateByTags(tags);

    // Pre-populate cache with updated data
    await this.cache.set(`product:${productId}`, product, {
      tags: [
        "product",
        `category:${product.categoryId}`,
        `brand:${product.brandId}`,
      ],
    });

    return product;
  }

  async searchProducts(
    query: ProductSearchQuery
  ): Promise<ProductSearchResult> {
    const cacheKey = this.buildSearchCacheKey(query);
    const tags = ["search", `category:${query.categoryId || "all"}`];

    const cached = await this.cache.get<ProductSearchResult>(cacheKey, tags);
    if (cached) {
      this.metrics.incrementCounter("search_cache_hit");
      return cached;
    }

    this.metrics.incrementCounter("search_cache_miss");

    const results = await this.database.searchProducts(query);

    // Cache search results with shorter TTL for dynamic data
    await this.cache.set(cacheKey, results, {
      ttl: 300000, // 5 minutes for search results
      tags,
    });

    return results;
  }

  private calculateProductTTL(product: Product): number {
    // Dynamic TTL based on product characteristics
    if (product.isNew || product.isOnSale) {
      return 300000; // 5 minutes for frequently changing products
    } else if (product.isPopular) {
      return 1800000; // 30 minutes for popular products
    } else {
      return 3600000; // 1 hour for regular products
    }
  }

  private buildSearchCacheKey(query: ProductSearchQuery): string {
    const keyParts = [
      "search",
      query.term || "all",
      query.categoryId || "all",
      query.brandId || "all",
      query.sortBy || "default",
      `page:${query.page || 1}`,
      `limit:${query.limit || 20}`,
    ];

    return keyParts.join(":");
  }
}

// Cache warming service for critical data
export class CacheWarmupService {
  constructor(
    private cache: IntelligentCacheManager,
    private productService: OptimizedProductService,
    private analytics: AnalyticsService
  ) {}

  async warmCriticalData(): Promise<void> {
    const warmupTasks = [
      this.warmPopularProducts(),
      this.warmTrendingSearches(),
      this.warmCategoryData(),
      this.warmHomepageData(),
    ];

    await Promise.all(warmupTasks);
  }

  private async warmPopularProducts(): Promise<void> {
    const popularProductIds = await this.analytics.getPopularProducts(100);

    const warmupPromises = popularProductIds.map(async (productId) => {
      try {
        await this.productService.getProduct(productId); // This will cache the product
      } catch (error) {
        console.error(`Failed to warm product ${productId}:`, error);
      }
    });

    await Promise.all(warmupPromises);
  }

  private async warmTrendingSearches(): Promise<void> {
    const trendingQueries = await this.analytics.getTrendingSearches(50);

    const warmupPromises = trendingQueries.map(async (query) => {
      try {
        await this.productService.searchProducts(query);
      } catch (error) {
        console.error(`Failed to warm search "${query.term}":`, error);
      }
    });

    await Promise.all(warmupPromises);
  }
}

// Supporting interfaces
interface CacheConfig {
  l1MaxEntries?: number;
  l1MaxAge?: number;
  redisNodes: string[];
}

interface CacheSetOptions {
  ttl?: number;
  tags?: string[];
}

interface InvalidationEvent {
  type: "tags" | "pattern";
  tags?: string[];
  pattern?: string;
  timestamp: number;
}

interface CacheWarmupConfig {
  dataLoader: (key: string) => Promise<any>;
  ttl?: number;
  tags?: string[];
}

// The async nightmare that destroys performance
class OrderProcessingService {
  async processOrder(orderData: OrderRequest) {
    // Sequential processing - blocking each step - RED FLAG #1
    const user = await this.validateUser(orderData.userId);

    // Each await blocks the next operation
    const inventory = await this.checkInventory(orderData.items); // 500ms
    const pricing = await this.calculatePricing(orderData.items); // 300ms
    const tax = await this.calculateTax(orderData.total, user.address); // 200ms
    const shipping = await this.calculateShipping(
      orderData.items,
      user.address
    ); // 400ms

    // More sequential operations - RED FLAG #2
    await this.reserveInventory(orderData.items); // 200ms
    await this.processPayment(orderData.payment); // 1000ms
    await this.createOrderRecord(orderData); // 300ms

    // Heavy CPU work on event loop - RED FLAG #3
    const recommendations = await this.generateRecommendations(
      user.id,
      orderData.items
    );

    // Non-critical operations blocking order completion - RED FLAG #4
    await this.sendConfirmationEmail(user.email, orderData);
    await this.updateAnalytics(orderData);
    await this.generateReceipt(orderData);

    // Total time: ~3+ seconds for operations that could be parallelized
    return { success: true };

    // Problems this creates:
    // - Event loop blocked by CPU-intensive operations
    // - Network I/O operations running in sequence instead of parallel
    // - Critical path blocked by non-critical operations
    // - Memory building up while long operations complete
    // - No error isolation - one failure kills the whole process
  }

  async generateRecommendations(userId: string, items: OrderItem[]) {
    // Synchronous CPU-intensive work blocking event loop - RED FLAG #5
    const userHistory = await this.getUserHistory(userId);

    // This calculation takes 2+ seconds and blocks everything
    let recommendations = [];
    for (const item of items) {
      for (const historyItem of userHistory) {
        // Nested loops - O(n²) - RED FLAG #6
        const similarity = this.calculateSimilarity(item, historyItem);
        if (similarity > 0.8) {
          const relatedProducts = await this.getRelatedProducts(historyItem.id); // DB call in loop - RED FLAG #7
          recommendations.push(...relatedProducts);
        }
      }
    }

    // More blocking synchronous processing
    recommendations = this.deduplicateRecommendations(recommendations);
    recommendations = this.sortByRelevance(recommendations, userHistory);

    return recommendations;
  }
}

// High-performance async processing with proper concurrency
export class OptimizedOrderProcessingService {
  private readonly workerPool: WorkerPool;
  private readonly semaphore: Semaphore;
  private readonly circuitBreaker: CircuitBreaker;

  constructor(
    private userService: UserService,
    private inventoryService: InventoryService,
    private pricingService: PricingService,
    private taxService: TaxService,
    private shippingService: ShippingService,
    private paymentService: PaymentService,
    private orderService: OrderService,
    private emailService: EmailService,
    private analyticsService: AnalyticsService,
    private metrics: IPerformanceMetrics
  ) {
    this.workerPool = new WorkerPool({
      maxWorkers: 4,
      workerScript: path.join(__dirname, "recommendation-worker.js"),
    });

    this.semaphore = new Semaphore(10); // Limit concurrent operations
    this.circuitBreaker = new CircuitBreaker({
      errorThreshold: 5,
      timeout: 30000,
      resetTimeout: 60000,
    });
  }

  async processOrder(orderData: OrderRequest): Promise<OrderResult> {
    const startTime = Date.now();
    const operationId = uuidv4().substring(0, 8);

    try {
      this.metrics.incrementCounter("order_processing_started");

      // Step 1: Validate user (required for all other operations)
      const user = await this.validateUser(orderData.userId);

      // Step 2: Run independent operations in parallel
      const [inventory, pricing, tax, shipping] = await Promise.all([
        this.checkInventoryWithRetry(orderData.items, operationId),
        this.calculatePricingWithRetry(orderData.items, operationId),
        this.calculateTaxWithRetry(orderData.total, user.address, operationId),
        this.calculateShippingWithRetry(
          orderData.items,
          user.address,
          operationId
        ),
      ]);

      // Step 3: Process critical path operations
      const criticalOperations = await this.processCriticalPath(
        orderData,
        user,
        inventory,
        pricing,
        operationId
      );

      // Step 4: Launch non-critical operations asynchronously (fire-and-forget)
      setImmediate(() => {
        this.processNonCriticalOperations(orderData, user, operationId).catch(
          (error) => {
            this.metrics.incrementCounter("non_critical_operation_error");
            console.error("Non-critical operation failed:", error);
          }
        );
      });

      const totalTime = Date.now() - startTime;
      this.metrics.recordTimer("order_processing_total", totalTime, {
        success: "true",
        operation_id: operationId,
      });

      return {
        success: true,
        orderId: criticalOperations.orderId,
        processingTime: totalTime,
      };
    } catch (error) {
      const totalTime = Date.now() - startTime;
      this.metrics.incrementCounter("order_processing_failed", {
        error_type: error.constructor.name,
        operation_id: operationId,
      });

      this.metrics.recordTimer("order_processing_total", totalTime, {
        success: "false",
        operation_id: operationId,
      });

      throw error;
    }
  }

  private async processCriticalPath(
    orderData: OrderRequest,
    user: User,
    inventory: InventoryCheck,
    pricing: PricingResult,
    operationId: string
  ): Promise<CriticalPathResult> {
    // Operations that must complete for order to succeed
    const [reservationResult, paymentResult] = await Promise.all([
      this.reserveInventoryWithCompensation(orderData.items, operationId),
      this.processPaymentWithRetry(orderData.payment, operationId),
    ]);

    // Create order record only after critical operations succeed
    const order = await this.createOrderRecord({
      ...orderData,
      inventoryReservation: reservationResult.reservationId,
      paymentTransaction: paymentResult.transactionId,
    });

    return {
      orderId: order.id,
      reservationId: reservationResult.reservationId,
      transactionId: paymentResult.transactionId,
    };
  }

  private async processNonCriticalOperations(
    orderData: OrderRequest,
    user: User,
    operationId: string
  ): Promise<void> {
    // These operations can fail without affecting order success
    const nonCriticalPromises = [
      this.sendConfirmationEmail(user.email, orderData, operationId),
      this.updateAnalytics(orderData, operationId),
      this.generateReceiptAsync(orderData, operationId),
      this.generateRecommendationsAsync(user.id, orderData.items, operationId),
    ];

    // Use allSettled to not fail if individual operations fail
    const results = await Promise.allSettled(nonCriticalPromises);

    results.forEach((result, index) => {
      if (result.status === "rejected") {
        this.metrics.incrementCounter("non_critical_operation_failed", {
          operation: ["email", "analytics", "receipt", "recommendations"][
            index
          ],
          operation_id: operationId,
        });
      }
    });
  }

  private async checkInventoryWithRetry(
    items: OrderItem[],
    operationId: string
  ): Promise<InventoryCheck> {
    return this.executeWithRetryAndCircuitBreaker(
      "inventory_check",
      async () => {
        const release = await this.semaphore.acquire();
        try {
          return await this.inventoryService.checkAvailability(items);
        } finally {
          release();
        }
      },
      { operationId, maxRetries: 3, backoffMs: 100 }
    );
  }

  private async processPaymentWithRetry(
    payment: PaymentData,
    operationId: string
  ): Promise<PaymentResult> {
    return this.executeWithRetryAndCircuitBreaker(
      "payment_processing",
      async () => {
        // Payment processing should not be retried automatically
        // Single attempt with proper error handling
        return await this.paymentService.processPayment(payment);
      },
      { operationId, maxRetries: 1, backoffMs: 0 }
    );
  }

  private async generateRecommendationsAsync(
    userId: string,
    items: OrderItem[],
    operationId: string
  ): Promise<void> {
    try {
      // Offload CPU-intensive work to worker thread
      const recommendations = await this.workerPool.execute(
        "generateRecommendations",
        {
          userId,
          items,
          operationId,
        }
      );

      // Store recommendations asynchronously
      await this.storeRecommendations(userId, recommendations);

      this.metrics.incrementCounter("recommendations_generated", {
        operation_id: operationId,
        recommendation_count: recommendations.length.toString(),
      });
    } catch (error) {
      this.metrics.incrementCounter("recommendation_generation_failed", {
        operation_id: operationId,
      });

      // Don't throw - recommendations are not critical
      console.error("Recommendation generation failed:", error);
    }
  }

  private async executeWithRetryAndCircuitBreaker<T>(
    operationName: string,
    operation: () => Promise<T>,
    options: { operationId: string; maxRetries: number; backoffMs: number }
  ): Promise<T> {
    return this.circuitBreaker.execute(async () => {
      let lastError: Error;

      for (let attempt = 1; attempt <= options.maxRetries; attempt++) {
        try {
          const startTime = Date.now();
          const result = await operation();

          this.metrics.recordTimer(
            `${operationName}_duration`,
            Date.now() - startTime,
            {
              attempt: attempt.toString(),
              success: "true",
              operation_id: options.operationId,
            }
          );

          if (attempt > 1) {
            this.metrics.incrementCounter(`${operationName}_retry_success`, {
              attempt: attempt.toString(),
              operation_id: options.operationId,
            });
          }

          return result;
        } catch (error) {
          lastError = error as Error;

          this.metrics.recordTimer(
            `${operationName}_duration`,
            Date.now() - Date.now(),
            {
              attempt: attempt.toString(),
              success: "false",
              operation_id: options.operationId,
            }
          );

          if (attempt < options.maxRetries) {
            const delay = options.backoffMs * Math.pow(2, attempt - 1);
            await this.sleep(delay);
          }
        }
      }

      throw lastError!;
    });
  }

  private sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms));
  }
}

// Advanced worker pool for CPU-intensive operations
export class WorkerPool {
  private workers: Worker[] = [];
  private availableWorkers: Worker[] = [];
  private taskQueue: TaskQueueItem[] = [];
  private readonly maxWorkers: number;

  constructor(config: WorkerPoolConfig) {
    this.maxWorkers = config.maxWorkers;
    this.initializeWorkers(config.workerScript);
  }

  private initializeWorkers(workerScript: string): void {
    for (let i = 0; i < this.maxWorkers; i++) {
      const worker = new Worker(workerScript);

      worker.on("message", (result: WorkerResult) => {
        this.handleWorkerResult(worker, result);
      });

      worker.on("error", (error: Error) => {
        this.handleWorkerError(worker, error);
      });

      this.workers.push(worker);
      this.availableWorkers.push(worker);
    }
  }

  async execute<T>(taskType: string, data: any): Promise<T> {
    return new Promise((resolve, reject) => {
      const task: TaskQueueItem = {
        id: uuidv4(),
        type: taskType,
        data,
        resolve,
        reject,
        timestamp: Date.now(),
      };

      if (this.availableWorkers.length > 0) {
        this.assignTaskToWorker(task);
      } else {
        this.taskQueue.push(task);
      }
    });
  }

  private assignTaskToWorker(task: TaskQueueItem): void {
    const worker = this.availableWorkers.pop()!;

    worker.postMessage({
      taskId: task.id,
      type: task.type,
      data: task.data,
    });

    // Store task reference for result handling
    (worker as any).currentTask = task;
  }

  private handleWorkerResult(worker: Worker, result: WorkerResult): void {
    const task = (worker as any).currentTask as TaskQueueItem;

    if (result.success) {
      task.resolve(result.data);
    } else {
      task.reject(new Error(result.error));
    }

    // Make worker available again
    delete (worker as any).currentTask;
    this.availableWorkers.push(worker);

    // Process next queued task
    if (this.taskQueue.length > 0) {
      const nextTask = this.taskQueue.shift()!;
      this.assignTaskToWorker(nextTask);
    }
  }

  private handleWorkerError(worker: Worker, error: Error): void {
    const task = (worker as any).currentTask as TaskQueueItem;

    if (task) {
      task.reject(error);
      delete (worker as any).currentTask;
    }

    // Remove failed worker and create a new one
    const index = this.workers.indexOf(worker);
    if (index > -1) {
      this.workers.splice(index, 1);
      worker.terminate();
    }

    // Create replacement worker
    const newWorker = new Worker("./recommendation-worker.js");
    this.workers.push(newWorker);
    this.availableWorkers.push(newWorker);
  }

  async terminate(): Promise<void> {
    const terminationPromises = this.workers.map((worker) =>
      worker.terminate()
    );
    await Promise.all(terminationPromises);
  }
}

// Circuit breaker for external service protection
export class CircuitBreaker {
  private failures = 0;
  private lastFailureTime = 0;
  private state: "CLOSED" | "OPEN" | "HALF_OPEN" = "CLOSED";

  constructor(private config: CircuitBreakerConfig) {}

  async execute<T>(operation: () => Promise<T>): Promise<T> {
    if (this.state === "OPEN") {
      if (Date.now() - this.lastFailureTime > this.config.resetTimeout) {
        this.state = "HALF_OPEN";
      } else {
        throw new Error("Circuit breaker is OPEN");
      }
    }

    try {
      const result = await Promise.race([operation(), this.timeoutPromise()]);

      if (this.state === "HALF_OPEN") {
        this.state = "CLOSED";
        this.failures = 0;
      }

      return result;
    } catch (error) {
      this.failures++;
      this.lastFailureTime = Date.now();

      if (this.failures >= this.config.errorThreshold) {
        this.state = "OPEN";
      }

      throw error;
    }
  }

  private timeoutPromise<T>(): Promise<T> {
    return new Promise((_, reject) => {
      setTimeout(() => {
        reject(new Error("Operation timeout"));
      }, this.config.timeout);
    });
  }

  getState(): string {
    return this.state;
  }
}

// Semaphore for controlling concurrency
export class Semaphore {
  private permits: number;
  private waitQueue: (() => void)[] = [];

  constructor(permits: number) {
    this.permits = permits;
  }

  async acquire(): Promise<() => void> {
    return new Promise((resolve) => {
      if (this.permits > 0) {
        this.permits--;
        resolve(() => this.release());
      } else {
        this.waitQueue.push(() => {
          this.permits--;
          resolve(() => this.release());
        });
      }
    });
  }

  private release(): void {
    this.permits++;

    if (this.waitQueue.length > 0) {
      const next = this.waitQueue.shift()!;
      next();
    }
  }
}

// Supporting interfaces
interface WorkerPoolConfig {
  maxWorkers: number;
  workerScript: string;
}

interface CircuitBreakerConfig {
  errorThreshold: number;
  timeout: number;
  resetTimeout: number;
}

interface TaskQueueItem {
  id: string;
  type: string;
  data: any;
  resolve: (value: any) => void;
  reject: (error: Error) => void;
  timestamp: number;
}

interface WorkerResult {
  success: boolean;
  data?: any;
  error?: string;
}

// The useless performance "testing" that teaches you nothing
describe("Product API Performance Tests", () => {
  test("should respond quickly", async () => {
    const start = Date.now();

    // Testing against empty database with no load - RED FLAG #1
    const response = await request(app).get("/api/products/123");

    const duration = Date.now() - start;

    // Arbitrary performance expectations - RED FLAG #2
    expect(response.status).toBe(200);
    expect(duration).toBeLessThan(100); // What does 100ms even mean?

    // Single request tells you nothing about real performance - RED FLAG #3
  });

  test("load test with 10 users", async () => {
    const promises = [];

    // Fake load that doesn't represent real usage - RED FLAG #4
    for (let i = 0; i < 10; i++) {
      promises.push(request(app).get("/api/products/123"));
    }

    const start = Date.now();
    await Promise.all(promises);
    const duration = Date.now() - start;

    // Testing unrealistic concurrent patterns - RED FLAG #5
    expect(duration).toBeLessThan(1000);

    // Problems with this approach:
    // - No realistic data volume
    // - No network latency simulation
    // - No database contention
    // - No memory pressure simulation
    // - No external service delays
    // - No cache warming period
    // - All requests identical (no variation)
    // - No measurement of percentiles, just averages
  });
});

// Realistic performance testing with proper measurement
export class PerformanceTestFramework {
  constructor(
    private app: Application,
    private database: TestDatabase,
    private cache: TestCache,
    private metrics: PerformanceMetrics
  ) {}

  async runPerformanceSuite(): Promise<PerformanceTestResults> {
    const testSuite: PerformanceTestSuite = {
      name: "Product API Performance Suite",
      setup: () => this.setupRealisticTestData(),
      teardown: () => this.cleanupTestData(),
      tests: [
        this.createBaselineTest(),
        this.createLoadTest(),
        this.createStressTest(),
        this.createSpikeTest(),
        this.createEnduranceTest(),
        this.createMemoryLeakTest(),
      ],
    };

    return await this.executeTestSuite(testSuite);
  }

  private createBaselineTest(): PerformanceTest {
    return {
      name: "Baseline Single User Performance",
      description: "Measure performance with no contention",
      config: {
        virtualUsers: 1,
        duration: "1m",
        rampUpTime: "10s",
        scenarios: [
          {
            name: "product_browsing",
            weight: 40,
            requests: [
              { method: "GET", path: "/api/products", weight: 30 },
              { method: "GET", path: "/api/products/:id", weight: 50 },
              { method: "GET", path: "/api/categories", weight: 20 },
            ],
          },
          {
            name: "search_flow",
            weight: 30,
            requests: [
              { method: "GET", path: "/api/search?q=laptop", weight: 60 },
              { method: "GET", path: "/api/products/:id", weight: 40 },
            ],
          },
          {
            name: "user_actions",
            weight: 30,
            requests: [
              { method: "POST", path: "/api/cart/add", weight: 25 },
              { method: "GET", path: "/api/cart", weight: 25 },
              { method: "POST", path: "/api/orders", weight: 25 },
              { method: "GET", path: "/api/orders/:id", weight: 25 },
            ],
          },
        ],
      },
      assertions: [
        { metric: "avg_response_time", threshold: 200 },
        { metric: "p95_response_time", threshold: 500 },
        { metric: "p99_response_time", threshold: 1000 },
        { metric: "error_rate", threshold: 0.01 },
      ],
    };
  }

  private createLoadTest(): PerformanceTest {
    return {
      name: "Normal Load Test",
      description: "Simulate expected production traffic",
      config: {
        virtualUsers: 100,
        duration: "10m",
        rampUpTime: "2m",
        scenarios: [
          {
            name: "typical_user_journey",
            weight: 100,
            requests: this.getTypicalUserJourney(),
          },
        ],
      },
      assertions: [
        { metric: "avg_response_time", threshold: 300 },
        { metric: "p95_response_time", threshold: 800 },
        { metric: "p99_response_time", threshold: 2000 },
        { metric: "error_rate", threshold: 0.05 },
        { metric: "throughput_per_second", threshold: 100 },
      ],
    };
  }

  private createStressTest(): PerformanceTest {
    return {
      name: "Stress Test - Breaking Point",
      description: "Find the breaking point of the system",
      config: {
        virtualUsers: 500,
        duration: "15m",
        rampUpTime: "5m",
        scenarios: [
          {
            name: "heavy_load_scenario",
            weight: 100,
            requests: this.getHeavyLoadScenario(),
          },
        ],
      },
      assertions: [
        { metric: "avg_response_time", threshold: 1000 },
        { metric: "p95_response_time", threshold: 3000 },
        { metric: "error_rate", threshold: 0.1 },
        { metric: "system_recovery_time", threshold: 30000 },
      ],
    };
  }

  private createSpikeTest(): PerformanceTest {
    return {
      name: "Spike Test - Traffic Surge",
      description: "Test system behavior during traffic spikes",
      config: {
        phases: [
          { virtualUsers: 50, duration: "2m" }, // Normal load
          { virtualUsers: 500, duration: "1m" }, // Spike
          { virtualUsers: 50, duration: "2m" }, // Recovery
          { virtualUsers: 1000, duration: "30s" }, // Larger spike
          { virtualUsers: 50, duration: "3m" }, // Recovery
        ],
        scenarios: [
          {
            name: "spike_scenario",
            weight: 100,
            requests: this.getSpikeScenario(),
          },
        ],
      },
      assertions: [
        { metric: "spike_response_degradation", threshold: 3.0 }, // Max 3x slower
        { metric: "recovery_time", threshold: 60000 }, // Recover within 1 minute
        { metric: "error_spike_ratio", threshold: 0.2 },
      ],
    };
  }

  private async executeTest(
    test: PerformanceTest
  ): Promise<PerformanceTestResult> {
    console.log(`Starting performance test: ${test.name}`);

    const testResult: PerformanceTestResult = {
      testName: test.name,
      startTime: Date.now(),
      endTime: 0,
      metrics: {},
      assertions: [],
      passed: false,
      details: {},
    };

    try {
      // Setup realistic test environment
      await this.setupTestEnvironment(test);

      // Execute the test
      const executionResult = await this.executeTestScenarios(test);

      // Collect metrics
      testResult.metrics = await this.collectMetrics(executionResult);

      // Validate assertions
      testResult.assertions = this.validateAssertions(
        test.assertions,
        testResult.metrics
      );
      testResult.passed = testResult.assertions.every((a) => a.passed);

      testResult.endTime = Date.now();

      // Generate detailed report
      testResult.details = this.generateTestDetails(
        executionResult,
        testResult.metrics
      );

      return testResult;
    } catch (error) {
      testResult.endTime = Date.now();
      testResult.error = error.message;
      testResult.passed = false;
      return testResult;
    } finally {
      await this.cleanupTestEnvironment();
    }
  }

  private async setupTestEnvironment(test: PerformanceTest): Promise<void> {
    // Create realistic database with proper data distribution
    await this.database.seed({
      products: 100000, // 100k products with realistic data sizes
      users: 50000, // 50k users with varying activity patterns
      orders: 500000, // 500k historical orders
      reviews: 200000, // 200k reviews with realistic text content
    });

    // Warm up caches with realistic access patterns
    await this.cache.warmup({
      popularProducts: 1000,
      trendingSearches: 500,
      userSessions: 5000,
    });

    // Simulate network latency
    await this.configureNetworkLatency({
      minLatency: 10, // 10ms minimum
      maxLatency: 100, // 100ms maximum
      distribution: "normal", // Normal distribution around mean
    });
  }

  private async executeTestScenarios(
    test: PerformanceTest
  ): Promise<TestExecutionResult> {
    const loadGenerator = new LoadGenerator({
      app: this.app,
      scenarios: test.config.scenarios,
      virtualUsers: test.config.virtualUsers,
      duration: test.config.duration,
      rampUpTime: test.config.rampUpTime,
    });

    // Start system monitoring
    const systemMonitor = this.startSystemMonitoring();

    try {
      // Execute load test
      const loadResults = await loadGenerator.execute();

      // Stop monitoring and collect system metrics
      const systemMetrics = await systemMonitor.stop();

      return {
        loadResults,
        systemMetrics,
        applicationMetrics: await this.collectApplicationMetrics(),
      };
    } catch (error) {
      await systemMonitor.stop();
      throw error;
    }
  }

  private validateAssertions(
    assertions: PerformanceAssertion[],
    metrics: PerformanceMetrics
  ): AssertionResult[] {
    return assertions.map((assertion) => {
      const actualValue = metrics[assertion.metric];
      const passed = actualValue <= assertion.threshold;

      return {
        metric: assertion.metric,
        expected: assertion.threshold,
        actual: actualValue,
        passed,
        message: passed
          ? `✓ ${assertion.metric}: ${actualValue} <= ${assertion.threshold}`
          : `✗ ${assertion.metric}: ${actualValue} > ${assertion.threshold}`,
      };
    });
  }

  private generatePerformanceReport(
    results: PerformanceTestResult[]
  ): PerformanceReport {
    return {
      timestamp: Date.now(),
      testSuite: "Product API Performance Suite",
      environment: process.env.NODE_ENV,
      systemInfo: this.getSystemInfo(),
      results,
      summary: {
        totalTests: results.length,
        passedTests: results.filter((r) => r.passed).length,
        failedTests: results.filter((r) => !r.passed).length,
        overallScore: this.calculatePerformanceScore(results),
      },
      recommendations: this.generatePerformanceRecommendations(results),
      trendAnalysis: this.compareToPreviousRuns(results),
    };
  }

  private generatePerformanceRecommendations(
    results: PerformanceTestResult[]
  ): string[] {
    const recommendations: string[] = [];

    results.forEach((result) => {
      // Analyze response time patterns
      if (result.metrics.p95_response_time > 1000) {
        recommendations.push(
          "Consider implementing response caching for slow endpoints"
        );
      }

      if (result.metrics.database_connection_pool_exhaustion > 0.8) {
        recommendations.push(
          "Increase database connection pool size or optimize query efficiency"
        );
      }

      if (result.metrics.memory_usage_peak > 0.9) {
        recommendations.push(
          "Investigate memory leaks and implement garbage collection optimization"
        );
      }

      if (result.metrics.cpu_usage_avg > 0.8) {
        recommendations.push(
          "Consider horizontal scaling or optimize CPU-intensive operations"
        );
      }

      if (result.metrics.error_rate > 0.05) {
        recommendations.push(
          "Improve error handling and implement circuit breakers for external services"
        );
      }
    });

    return [...new Set(recommendations)]; // Remove duplicates
  }
}

// Realistic load generation with proper user behavior simulation
export class LoadGenerator {
  private virtualUsers: VirtualUser[] = [];
  private scenarios: LoadTestScenario[];
  private metrics: LoadTestMetrics;

  constructor(private config: LoadGeneratorConfig) {
    this.scenarios = config.scenarios;
    this.metrics = new LoadTestMetrics();
  }

  async execute(): Promise<LoadTestResult> {
    console.log(
      `Starting load generation with ${this.config.virtualUsers} virtual users`
    );

    // Ramp up virtual users gradually
    await this.rampUpUsers();

    // Execute test for specified duration
    const testPromise = this.runTestDuration();

    // Monitor system resources during test
    const monitoringPromise = this.monitorSystemResources();

    // Wait for test completion
    await Promise.race([testPromise, monitoringPromise]);

    // Ramp down users
    await this.rampDownUsers();

    return this.metrics.getResults();
  }

  private async createVirtualUser(userId: number): Promise<VirtualUser> {
    return new VirtualUser(userId, {
      app: this.config.app,
      scenarios: this.scenarios,
      thinkTime: this.getRealisticThinkTime(),
      sessionDuration: this.getRealisticSessionDuration(),
      behaviorProfile: this.generateUserBehaviorProfile(),
    });
  }

  private getRealisticThinkTime(): ThinkTimeConfig {
    // Simulate realistic user behavior with pauses
    return {
      min: 1000, // 1 second minimum pause
      max: 10000, // 10 seconds maximum pause
      distribution: "lognormal", // Most pauses are short, some are long
      mean: 3000, // 3 second average
    };
  }

  private generateUserBehaviorProfile(): UserBehaviorProfile {
    // Simulate different user types
    const profiles = [
      { type: "browser", searchIntensive: true, purchaseProbability: 0.1 },
      { type: "buyer", searchIntensive: false, purchaseProbability: 0.8 },
      { type: "researcher", searchIntensive: true, purchaseProbability: 0.05 },
      { type: "returner", searchIntensive: false, purchaseProbability: 0.3 },
    ];

    return profiles[Math.floor(Math.random() * profiles.length)];
  }
}

// Comprehensive system monitoring during tests
export class SystemMonitor {
  private monitors: Monitor[] = [];
  private isRunning = false;
  private metrics: SystemMetrics = {};

  start(): void {
    this.isRunning = true;

    // CPU monitoring
    this.monitors.push(this.startCPUMonitoring());

    // Memory monitoring
    this.monitors.push(this.startMemoryMonitoring());

    // Database monitoring
    this.monitors.push(this.startDatabaseMonitoring());

    // Network monitoring
    this.monitors.push(this.startNetworkMonitoring());

    // Custom application metrics
    this.monitors.push(this.startApplicationMonitoring());
  }

  async stop(): Promise<SystemMetrics> {
    this.isRunning = false;

    // Stop all monitors
    await Promise.all(this.monitors.map((monitor) => monitor.stop()));

    return this.metrics;
  }

  private startCPUMonitoring(): Monitor {
    return {
      stop: setInterval(() => {
        if (!this.isRunning) return;

        const cpuUsage = process.cpuUsage();
        this.metrics.cpu = {
          user: cpuUsage.user,
          system: cpuUsage.system,
          utilization: (cpuUsage.user + cpuUsage.system) / 1000000, // Convert to percentage
        };
      }, 1000),
    };
  }

  private startMemoryMonitoring(): Monitor {
    return {
      stop: setInterval(() => {
        if (!this.isRunning) return;

        const memUsage = process.memoryUsage();
        this.metrics.memory = {
          heapUsed: memUsage.heapUsed,
          heapTotal: memUsage.heapTotal,
          external: memUsage.external,
          rss: memUsage.rss,
          utilization: (memUsage.heapUsed / memUsage.heapTotal) * 100,
        };
      }, 1000),
    };
  }
}

// Supporting interfaces and types
interface PerformanceTest {
  name: string;
  description: string;
  config: TestConfig;
  assertions: PerformanceAssertion[];
}

interface PerformanceAssertion {
  metric: string;
  threshold: number;
  operator?: "lt" | "lte" | "gt" | "gte" | "eq";
}

interface PerformanceTestResult {
  testName: string;
  startTime: number;
  endTime: number;
  metrics: PerformanceMetrics;
  assertions: AssertionResult[];
  passed: boolean;
  details?: any;
  error?: string;
}