// Your perfectly cached CRUD operations work great
const getArticles = async (page = 1) => {
  const cacheKey = `articles:page:${page}`;
  let articles = await cache.get(cacheKey);

  if (!articles) {
    articles = await db.articles
      .find({ published: true })
      .sort({ publishedAt: -1 })
      .skip((page - 1) * 20)
      .limit(20);
    await cache.set(cacheKey, articles, 300);
  }

  return articles;
};

// But users expect so much more:
// - "Search for articles about React hooks" (full-text search across millions of articles)
// - "Notify me when my followed authors publish" (real-time event processing)
// - "Generate my weekly digest" (background job processing with ML recommendations)
// - "Sync my reading progress across devices" (data synchronization patterns)
// - "Show trending topics" (streaming analytics on user behavior)

// Your static queries can't deliver modern expectations

// What users type: "articles about machine learning for beginners"
// What your database can do:
const searchResults = await db.articles.find({
  title: { $regex: /machine learning|beginners/i },
}); // Finds 2 relevant articles out of 50 matches

// What users expect:
// - Semantic search understanding "ML", "AI", "neural networks" relate to "machine learning"
// - Personalized results based on reading level and interests
// - Related articles and trending topics
// - Instant autocomplete suggestions
// - Faceted filtering by author, date, difficulty, topic clusters
// - Search result ranking based on quality signals and user preferences

// The gap between database capabilities and user expectations is massive

// ❌ Database-based search: Doesn't scale or satisfy users
const searchArticles = async (query, page = 1) => {
  // Simple regex search - slow and limited
  const results = await db.articles
    .find({
      $or: [
        { title: { $regex: query, $options: "i" } },
        { content: { $regex: query, $options: "i" } },
        { tags: { $in: [query] } },
      ],
      published: true,
    })
    .sort({ publishedAt: -1 })
    .skip((page - 1) * 20)
    .limit(20);

  return results;

  // Problems with database search:
  // - No relevance scoring (chronological order only)
  // - Regex queries are slow on large datasets
  // - No typo tolerance ("JavaScrip" won't find "JavaScript")
  // - No semantic understanding ("ML" won't find "Machine Learning")
  // - No faceted filtering or aggregations
  // - Full table scans kill database performance
  // - Can't handle complex queries like "React tutorials for intermediate developers"
};

// User searches for "node js performance optimization"
// Database finds 3 articles containing all those exact words
// Misses 50 highly relevant articles using terms like:
// - "Node.js" (punctuation difference)
// - "NodeJS" (spacing difference)
// - "server-side JavaScript" (semantic match)
// - "V8 engine optimization" (related concept)
// - "Express.js speed improvements" (related framework)

// ✅ Elasticsearch: Professional search that users love
const { Client } = require("@elastic/elasticsearch");

class SearchService {
  constructor() {
    this.client = new Client({
      node: process.env.ELASTICSEARCH_URL,
      auth: {
        username: process.env.ES_USERNAME,
        password: process.env.ES_PASSWORD,
      },
      requestTimeout: 30000,
      pingTimeout: 30000,
      sniffOnStart: true,
    });

    this.articlesIndex = "articles";
    this.setupIndex();
  }

  async setupIndex() {
    // Create index with sophisticated mapping
    const indexMapping = {
      mappings: {
        properties: {
          title: {
            type: "text",
            analyzer: "standard",
            fields: {
              exact: { type: "keyword" },
              autocomplete: {
                type: "search_as_you_type",
              },
            },
          },
          content: {
            type: "text",
            analyzer: "standard",
          },
          summary: {
            type: "text",
            analyzer: "standard",
          },
          author: {
            type: "object",
            properties: {
              id: { type: "keyword" },
              name: {
                type: "text",
                fields: {
                  exact: { type: "keyword" },
                },
              },
            },
          },
          tags: {
            type: "keyword", // Exact matching for facets
          },
          categories: {
            type: "keyword",
          },
          difficulty: {
            type: "keyword",
          },
          publishedAt: {
            type: "date",
          },
          updatedAt: {
            type: "date",
          },
          readingTime: {
            type: "integer",
          },
          viewCount: {
            type: "long",
          },
          likeCount: {
            type: "long",
          },
          commentCount: {
            type: "long",
          },
          qualityScore: {
            type: "float", // Custom ranking signal
          },
        },
      },
      settings: {
        analysis: {
          analyzer: {
            content_analyzer: {
              type: "custom",
              tokenizer: "standard",
              filter: ["lowercase", "stop", "snowball", "synonym_filter"],
            },
          },
          filter: {
            synonym_filter: {
              type: "synonym",
              synonyms: [
                "js,javascript",
                "nodejs,node.js,node",
                "ml,machine learning,artificial intelligence,ai",
                "react,reactjs",
                "vue,vuejs",
                "db,database",
              ],
            },
          },
        },
      },
    };

    try {
      await this.client.indices.create({
        index: this.articlesIndex,
        body: indexMapping,
      });
    } catch (error) {
      if (
        error.meta?.body?.error?.type !== "resource_already_exists_exception"
      ) {
        console.error("Failed to create index:", error);
      }
    }
  }

  async indexArticle(article) {
    // Calculate quality score based on engagement metrics
    const qualityScore = this.calculateQualityScore(article);

    const document = {
      id: article.id,
      title: article.title,
      content: article.content,
      summary: article.summary,
      author: {
        id: article.author.id,
        name: article.author.name,
      },
      tags: article.tags || [],
      categories: article.categories || [],
      difficulty: article.difficulty || "intermediate",
      publishedAt: article.publishedAt,
      updatedAt: article.updatedAt,
      readingTime: article.readingTime || 5,
      viewCount: article.viewCount || 0,
      likeCount: article.likeCount || 0,
      commentCount: article.commentCount || 0,
      qualityScore,
    };

    await this.client.index({
      index: this.articlesIndex,
      id: article.id,
      body: document,
      refresh: "wait_for", // Make searchable immediately
    });
  }

  calculateQualityScore(article) {
    // Sophisticated scoring algorithm
    let score = 50; // Base score

    // Content quality indicators
    if (article.content && article.content.length > 2000) score += 10;
    if (article.summary && article.summary.length > 100) score += 5;
    if (article.tags && article.tags.length >= 3) score += 5;

    // Engagement signals
    const engagementRate =
      (article.likeCount || 0) / Math.max(article.viewCount || 1, 1);
    score += Math.min(engagementRate * 100, 25);

    // Recency boost
    const daysSincePublished =
      (Date.now() - new Date(article.publishedAt).getTime()) /
      (1000 * 60 * 60 * 24);
    if (daysSincePublished < 30)
      score += Math.max(10 - daysSincePublished / 3, 0);

    // Author reputation
    if (article.author.reputation > 1000) score += 10;

    return Math.min(Math.max(score, 0), 100);
  }

  async search(query, options = {}) {
    const {
      page = 1,
      limit = 20,
      categories = [],
      tags = [],
      authors = [],
      difficulty = null,
      minReadingTime = null,
      maxReadingTime = null,
      sortBy = "relevance",
    } = options;

    // Build complex search query
    const searchQuery = {
      bool: {
        must: [],
        filter: [],
        should: [],
        minimum_should_match: 1,
      },
    };

    // Main text search with boosting
    if (query) {
      searchQuery.bool.must.push({
        multi_match: {
          query,
          fields: [
            "title^3", // Title matches are most important
            "summary^2", // Summary matches are second
            "content", // Content matches are standard
            "tags^1.5", // Tag matches are slightly boosted
            "author.name", // Author name matches
          ],
          fuzziness: "AUTO", // Handle typos
          operator: "or",
        },
      });

      // Boost exact phrase matches in title
      searchQuery.bool.should.push({
        match_phrase: {
          title: {
            query,
            boost: 5,
          },
        },
      });

      // Boost prefix matches for autocomplete feel
      searchQuery.bool.should.push({
        prefix: {
          "title.autocomplete": {
            value: query.split(" ").pop(), // Last word for autocomplete
            boost: 2,
          },
        },
      });
    }

    // Apply filters
    if (categories.length > 0) {
      searchQuery.bool.filter.push({
        terms: { categories },
      });
    }

    if (tags.length > 0) {
      searchQuery.bool.filter.push({
        terms: { tags },
      });
    }

    if (authors.length > 0) {
      searchQuery.bool.filter.push({
        terms: { "author.id": authors },
      });
    }

    if (difficulty) {
      searchQuery.bool.filter.push({
        term: { difficulty },
      });
    }

    if (minReadingTime || maxReadingTime) {
      const rangeFilter = { range: { readingTime: {} } };
      if (minReadingTime) rangeFilter.range.readingTime.gte = minReadingTime;
      if (maxReadingTime) rangeFilter.range.readingTime.lte = maxReadingTime;
      searchQuery.bool.filter.push(rangeFilter);
    }

    // Build sort configuration
    let sort = [];
    switch (sortBy) {
      case "newest":
        sort = [{ publishedAt: { order: "desc" } }];
        break;
      case "popular":
        sort = [{ viewCount: { order: "desc" } }];
        break;
      case "trending":
        sort = [
          { likeCount: { order: "desc" } },
          { viewCount: { order: "desc" } },
        ];
        break;
      case "relevance":
      default:
        sort = [
          "_score",
          { qualityScore: { order: "desc" } },
          { publishedAt: { order: "desc" } },
        ];
        break;
    }

    // Execute search with aggregations for facets
    const response = await this.client.search({
      index: this.articlesIndex,
      body: {
        query: searchQuery,
        sort,
        from: (page - 1) * limit,
        size: limit,
        highlight: {
          fields: {
            title: { number_of_fragments: 0 },
            content: {
              fragment_size: 150,
              number_of_fragments: 2,
            },
            summary: { number_of_fragments: 0 },
          },
          pre_tags: ["<mark>"],
          post_tags: ["</mark>"],
        },
        aggs: {
          categories: {
            terms: { field: "categories", size: 20 },
          },
          tags: {
            terms: { field: "tags", size: 50 },
          },
          authors: {
            terms: { field: "author.name.exact", size: 20 },
          },
          difficulty: {
            terms: { field: "difficulty" },
          },
          reading_time_ranges: {
            range: {
              field: "readingTime",
              ranges: [
                { key: "quick", to: 5 },
                { key: "medium", from: 5, to: 15 },
                { key: "long", from: 15 },
              ],
            },
          },
        },
      },
    });

    // Format results with highlighting and metadata
    const results = response.body.hits.hits.map((hit) => {
      const article = hit._source;
      article.searchScore = hit._score;

      // Include highlighted snippets
      if (hit.highlight) {
        article.highlights = {
          title: hit.highlight.title?.[0],
          content: hit.highlight.content || [],
          summary: hit.highlight.summary?.[0],
        };
      }

      return article;
    });

    return {
      results,
      total: response.body.hits.total.value,
      page,
      limit,
      facets: {
        categories: response.body.aggregations.categories.buckets,
        tags: response.body.aggregations.tags.buckets,
        authors: response.body.aggregations.authors.buckets,
        difficulty: response.body.aggregations.difficulty.buckets,
        readingTimeRanges:
          response.body.aggregations.reading_time_ranges.buckets,
      },
      searchTime: response.body.took,
    };
  }

  async autocomplete(query, limit = 10) {
    if (!query || query.length < 2) return [];

    const response = await this.client.search({
      index: this.articlesIndex,
      body: {
        query: {
          bool: {
            should: [
              {
                match_phrase_prefix: {
                  "title.autocomplete": {
                    query,
                    max_expansions: 10,
                  },
                },
              },
              {
                match_phrase_prefix: {
                  tags: {
                    query,
                    boost: 0.5,
                  },
                },
              },
            ],
          },
        },
        _source: ["title", "author.name"],
        size: limit,
      },
    });

    return response.body.hits.hits.map((hit) => ({
      title: hit._source.title,
      author: hit._source.author.name,
    }));
  }
}

// Usage in API endpoints
const searchService = new SearchService();

app.get("/search", async (req, res) => {
  try {
    const {
      q: query,
      page = 1,
      limit = 20,
      categories,
      tags,
      authors,
      difficulty,
      minReadingTime,
      maxReadingTime,
      sort = "relevance",
    } = req.query;

    const options = {
      page: parseInt(page),
      limit: parseInt(limit),
      categories: categories ? categories.split(",") : [],
      tags: tags ? tags.split(",") : [],
      authors: authors ? authors.split(",") : [],
      difficulty,
      minReadingTime: minReadingTime ? parseInt(minReadingTime) : null,
      maxReadingTime: maxReadingTime ? parseInt(maxReadingTime) : null,
      sortBy: sort,
    };

    const results = await searchService.search(query, options);

    res.json({
      success: true,
      ...results,
    });
  } catch (error) {
    console.error("Search error:", error);
    res.status(500).json({
      error: "Search failed",
      message: error.message,
    });
  }
});

app.get("/search/autocomplete", async (req, res) => {
  try {
    const { q: query, limit = 10 } = req.query;
    const suggestions = await searchService.autocomplete(
      query,
      parseInt(limit)
    );
    res.json({ suggestions });
  } catch (error) {
    console.error("Autocomplete error:", error);
    res.status(500).json({ error: "Autocomplete failed" });
  }
});

// Advanced semantic search with embeddings
const { OpenAI } = require("openai");

class SemanticSearchService extends SearchService {
  constructor() {
    super();
    this.openai = new OpenAI({
      apiKey: process.env.OPENAI_API_KEY,
    });
    this.embeddingsIndex = "article_embeddings";
    this.setupEmbeddingsIndex();
  }

  async setupEmbeddingsIndex() {
    const mappings = {
      mappings: {
        properties: {
          article_id: { type: "keyword" },
          title_embedding: {
            type: "dense_vector",
            dims: 1536, // OpenAI ada-002 embedding size
          },
          content_embedding: {
            type: "dense_vector",
            dims: 1536,
          },
          created_at: { type: "date" },
        },
      },
    };

    try {
      await this.client.indices.create({
        index: this.embeddingsIndex,
        body: mappings,
      });
    } catch (error) {
      if (
        error.meta?.body?.error?.type !== "resource_already_exists_exception"
      ) {
        console.error("Failed to create embeddings index:", error);
      }
    }
  }

  async generateEmbeddings(text) {
    const response = await this.openai.embeddings.create({
      model: "text-embedding-ada-002",
      input: text,
    });

    return response.data[0].embedding;
  }

  async indexArticleWithEmbeddings(article) {
    // Index in traditional search first
    await this.indexArticle(article);

    // Generate embeddings for semantic search
    const titleEmbedding = await this.generateEmbeddings(article.title);

    // Use first 1000 chars of content for embedding to stay within token limits
    const contentPreview = article.content.substring(0, 1000);
    const contentEmbedding = await this.generateEmbeddings(contentPreview);

    await this.client.index({
      index: this.embeddingsIndex,
      id: article.id,
      body: {
        article_id: article.id,
        title_embedding: titleEmbedding,
        content_embedding: contentEmbedding,
        created_at: new Date(),
      },
    });
  }

  async semanticSearch(query, options = {}) {
    const { limit = 20, threshold = 0.7 } = options;

    // Generate embedding for the query
    const queryEmbedding = await this.generateEmbeddings(query);

    // Search using cosine similarity
    const response = await this.client.search({
      index: this.embeddingsIndex,
      body: {
        query: {
          script_score: {
            query: { match_all: {} },
            script: {
              source:
                "cosineSimilarity(params.query_vector, 'content_embedding') + 1.0",
              params: {
                query_vector: queryEmbedding,
              },
            },
          },
        },
        size: limit,
        min_score: threshold,
      },
    });

    // Get article details for matching embeddings
    const articleIds = response.body.hits.hits.map(
      (hit) => hit._source.article_id
    );

    if (articleIds.length === 0) {
      return { results: [], total: 0 };
    }

    const articlesResponse = await this.client.search({
      index: this.articlesIndex,
      body: {
        query: {
          terms: { id: articleIds },
        },
        size: limit,
      },
    });

    // Combine semantic scores with article data
    const semanticScores = new Map();
    response.body.hits.hits.forEach((hit) => {
      semanticScores.set(hit._source.article_id, hit._score);
    });

    const results = articlesResponse.body.hits.hits
      .map((hit) => {
        const article = hit._source;
        article.semanticScore = semanticScores.get(article.id);
        return article;
      })
      .sort((a, b) => b.semanticScore - a.semanticScore);

    return {
      results,
      total: results.length,
      searchType: "semantic",
    };
  }

  async hybridSearch(query, options = {}) {
    // Combine traditional and semantic search
    const [traditionalResults, semanticResults] = await Promise.all([
      this.search(query, { ...options, limit: options.limit * 2 }),
      this.semanticSearch(query, { ...options, limit: options.limit * 2 }),
    ]);

    // Merge and rank results using hybrid scoring
    const mergedResults = this.mergeSearchResults(
      traditionalResults.results,
      semanticResults.results,
      options.hybridWeight || 0.5
    );

    return {
      results: mergedResults.slice(0, options.limit || 20),
      total: Math.max(traditionalResults.total, semanticResults.total),
      searchType: "hybrid",
      breakdown: {
        traditional: traditionalResults.total,
        semantic: semanticResults.total,
      },
    };
  }

  mergeSearchResults(traditional, semantic, semanticWeight = 0.5) {
    const traditionalWeight = 1 - semanticWeight;
    const articleMap = new Map();

    // Add traditional results
    traditional.forEach((article, index) => {
      const rank = index + 1;
      const score = (article.searchScore || 0) * traditionalWeight;

      articleMap.set(article.id, {
        ...article,
        hybridScore: score,
        traditionalRank: rank,
        semanticRank: null,
      });
    });

    // Merge semantic results
    semantic.forEach((article, index) => {
      const rank = index + 1;
      const score = (article.semanticScore || 0) * semanticWeight;

      if (articleMap.has(article.id)) {
        // Article exists in both results - boost score
        const existing = articleMap.get(article.id);
        existing.hybridScore += score;
        existing.semanticRank = rank;
      } else {
        // Semantic-only result
        articleMap.set(article.id, {
          ...article,
          hybridScore: score,
          traditionalRank: null,
          semanticRank: rank,
        });
      }
    });

    // Sort by hybrid score and return
    return Array.from(articleMap.values()).sort(
      (a, b) => b.hybridScore - a.hybridScore
    );
  }
}

// User expectations across devices:
// Phone: Mark article as read
// Laptop: Article should appear as read immediately
// Tablet: Reading progress should be synced
// Web app: Bookmarks should be updated

// ❌ Naive polling approach: Wasteful and slow
setInterval(async () => {
  // Poll server every 30 seconds from every device
  const updates = await fetch("/api/sync-check");
  if (updates.hasChanges) {
    await refreshLocalData();
  }
}, 30000);

// Problems:
// - Constant unnecessary requests
// - 30-second delay for updates
// - Server overloaded with polling requests
// - Battery drain on mobile devices
// - Doesn't scale beyond a few thousand users

// ✅ Event-driven synchronization with WebSockets
class DataSyncManager {
  constructor(userId, deviceId) {
    this.userId = userId;
    this.deviceId = deviceId;
    this.socket = null;
    this.localStore = new Map();
    this.pendingSync = new Set();
    this.conflictResolver = new ConflictResolver();

    this.connect();
    this.setupOfflineHandling();
  }

  connect() {
    this.socket = io("/sync", {
      auth: {
        userId: this.userId,
        deviceId: this.deviceId,
      },
      transports: ["websocket", "polling"],
    });

    this.socket.on("connect", () => {
      console.log("Sync connected");
      this.syncPendingChanges();
    });

    this.socket.on("data_update", (event) => {
      this.handleRemoteUpdate(event);
    });

    this.socket.on("sync_conflict", (conflict) => {
      this.handleSyncConflict(conflict);
    });

    this.socket.on("disconnect", () => {
      console.log("Sync disconnected");
      this.switchToOfflineMode();
    });
  }

  // Optimistic updates with conflict resolution
  async updateData(collection, id, changes, options = {}) {
    const timestamp = Date.now();
    const changeId = generateId();

    // Apply optimistically to local state
    const localItem = this.localStore.get(`${collection}:${id}`) || {};
    const updatedItem = {
      ...localItem,
      ...changes,
      _lastModified: timestamp,
      _deviceId: this.deviceId,
      _changeId: changeId,
    };

    this.localStore.set(`${collection}:${id}`, updatedItem);

    // Update UI immediately
    this.emitLocalChange(collection, id, updatedItem);

    // Queue for sync
    this.pendingSync.add({
      collection,
      id,
      changes,
      timestamp,
      changeId,
      operation: "update",
    });

    try {
      // Send to server
      await this.syncChange({
        collection,
        id,
        changes,
        timestamp,
        changeId,
        deviceId: this.deviceId,
        userId: this.userId,
      });

      // Remove from pending if successful
      this.pendingSync.delete(changeId);
    } catch (error) {
      console.error("Sync failed:", error);
      // Keep in pending for retry
      this.scheduleRetry(changeId);
    }

    return updatedItem;
  }

  async syncChange(change) {
    if (this.socket?.connected) {
      // Real-time sync via WebSocket
      return new Promise((resolve, reject) => {
        this.socket.emit("sync_change", change, (response) => {
          if (response.success) {
            resolve(response);
          } else {
            reject(new Error(response.error));
          }
        });
      });
    } else {
      // Fallback to HTTP API
      const response = await fetch("/api/sync", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify(change),
      });

      if (!response.ok) {
        throw new Error(`Sync failed: ${response.statusText}`);
      }

      return response.json();
    }
  }

  handleRemoteUpdate(event) {
    const { collection, id, data, timestamp, deviceId, changeId } = event;

    // Ignore updates from this device
    if (deviceId === this.deviceId) return;

    const localKey = `${collection}:${id}`;
    const localItem = this.localStore.get(localKey);

    if (!localItem) {
      // New item from remote
      this.localStore.set(localKey, data);
      this.emitLocalChange(collection, id, data);
      return;
    }

    // Check for conflicts
    if (localItem._lastModified > timestamp) {
      // Local version is newer - potential conflict
      this.handleSyncConflict({
        collection,
        id,
        local: localItem,
        remote: data,
        localTimestamp: localItem._lastModified,
        remoteTimestamp: timestamp,
      });
      return;
    }

    // Remote version is newer or equal - accept it
    this.localStore.set(localKey, data);
    this.emitLocalChange(collection, id, data);
  }

  handleSyncConflict(conflict) {
    const resolution = this.conflictResolver.resolve(conflict);

    switch (resolution.strategy) {
      case "merge":
        // Merge both versions
        const merged = this.mergeChanges(conflict.local, conflict.remote);
        this.localStore.set(`${conflict.collection}:${conflict.id}`, merged);
        this.emitLocalChange(conflict.collection, conflict.id, merged);

        // Send merged version to server
        this.syncChange({
          collection: conflict.collection,
          id: conflict.id,
          changes: merged,
          timestamp: Date.now(),
          changeId: generateId(),
          deviceId: this.deviceId,
          userId: this.userId,
          isConflictResolution: true,
        });
        break;

      case "local_wins":
        // Keep local version, inform server
        this.syncChange({
          collection: conflict.collection,
          id: conflict.id,
          changes: conflict.local,
          timestamp: Date.now(),
          changeId: generateId(),
          deviceId: this.deviceId,
          userId: this.userId,
          isConflictResolution: true,
        });
        break;

      case "remote_wins":
        // Accept remote version
        this.localStore.set(
          `${conflict.collection}:${conflict.id}`,
          conflict.remote
        );
        this.emitLocalChange(conflict.collection, conflict.id, conflict.remote);
        break;

      case "user_choice":
        // Present conflict to user for manual resolution
        this.emitConflictForUser(conflict);
        break;
    }
  }

  mergeChanges(local, remote) {
    // Implement field-level merging based on data type
    const merged = { ...remote }; // Start with remote as base

    // Merge arrays by combining unique items
    Object.keys(local).forEach((key) => {
      if (Array.isArray(local[key]) && Array.isArray(remote[key])) {
        merged[key] = [...new Set([...remote[key], ...local[key]])];
      } else if (local._lastModified > remote._lastModified) {
        // Local field is newer
        merged[key] = local[key];
      }
    });

    merged._lastModified = Math.max(
      local._lastModified || 0,
      remote._lastModified || 0
    );

    return merged;
  }

  // Offline handling
  setupOfflineHandling() {
    window.addEventListener("online", () => {
      console.log("Back online - syncing pending changes");
      this.syncPendingChanges();
    });

    window.addEventListener("offline", () => {
      console.log("Gone offline - queuing changes locally");
      this.switchToOfflineMode();
    });
  }

  async syncPendingChanges() {
    if (this.pendingSync.size === 0) return;

    const changes = Array.from(this.pendingSync);
    console.log(`Syncing ${changes.length} pending changes`);

    for (const change of changes) {
      try {
        await this.syncChange(change);
        this.pendingSync.delete(change.changeId);
      } catch (error) {
        console.error("Failed to sync pending change:", error);
        // Will retry later
      }
    }
  }

  switchToOfflineMode() {
    // Save all pending changes to localStorage for persistence
    localStorage.setItem(
      "pendingSync",
      JSON.stringify(Array.from(this.pendingSync))
    );

    // Switch UI to offline mode
    this.emitOfflineStatus(true);
  }

  // Event emitters for UI updates
  emitLocalChange(collection, id, data) {
    window.dispatchEvent(
      new CustomEvent("dataSync:change", {
        detail: { collection, id, data },
      })
    );
  }

  emitConflictForUser(conflict) {
    window.dispatchEvent(
      new CustomEvent("dataSync:conflict", {
        detail: conflict,
      })
    );
  }

  emitOfflineStatus(isOffline) {
    window.dispatchEvent(
      new CustomEvent("dataSync:offline", {
        detail: { isOffline },
      })
    );
  }
}

// Conflict resolution strategies
class ConflictResolver {
  resolve(conflict) {
    const { collection, local, remote } = conflict;

    // Collection-specific resolution strategies
    switch (collection) {
      case "user_preferences":
        return { strategy: "merge" };

      case "reading_progress":
        // Always take the furthest progress
        const localProgress = local.progress || 0;
        const remoteProgress = remote.progress || 0;
        return {
          strategy:
            localProgress > remoteProgress ? "local_wins" : "remote_wins",
        };

      case "bookmarks":
        return { strategy: "merge" }; // Combine bookmarks from both devices

      case "user_content":
        // User-created content requires manual resolution
        return { strategy: "user_choice" };

      default:
        // Default: most recent wins
        return {
          strategy:
            local._lastModified > remote._lastModified
              ? "local_wins"
              : "remote_wins",
        };
    }
  }
}

// Server-side sync handler
class SyncServer {
  constructor(io) {
    this.io = io;
    this.setupNamespace();
    this.userDevices = new Map(); // Track user devices
  }

  setupNamespace() {
    const syncNs = this.io.of("/sync");

    syncNs.use(async (socket, next) => {
      // Authenticate user
      const { userId, deviceId } = socket.handshake.auth;
      if (!userId || !deviceId) {
        return next(new Error("Authentication required"));
      }

      socket.userId = userId;
      socket.deviceId = deviceId;
      next();
    });

    syncNs.on("connection", (socket) => {
      this.handleConnection(socket);
    });
  }

  handleConnection(socket) {
    const { userId, deviceId } = socket;
    console.log(`User ${userId} connected from device ${deviceId}`);

    // Join user-specific room
    socket.join(`user:${userId}`);

    // Track device
    if (!this.userDevices.has(userId)) {
      this.userDevices.set(userId, new Set());
    }
    this.userDevices.get(userId).add(deviceId);

    socket.on("sync_change", async (change, callback) => {
      try {
        await this.handleSyncChange(socket, change);
        callback({ success: true });
      } catch (error) {
        console.error("Sync change failed:", error);
        callback({ success: false, error: error.message });
      }
    });

    socket.on("disconnect", () => {
      console.log(`User ${userId} disconnected from device ${deviceId}`);
      const userDevices = this.userDevices.get(userId);
      if (userDevices) {
        userDevices.delete(deviceId);
        if (userDevices.size === 0) {
          this.userDevices.delete(userId);
        }
      }
    });
  }

  async handleSyncChange(socket, change) {
    const { collection, id, changes, timestamp, deviceId, userId } = change;

    // Store change in database with conflict detection
    const existing = await db.collection(collection).findOne({
      id,
      userId,
    });

    if (
      existing &&
      existing._lastModified > timestamp &&
      !change.isConflictResolution
    ) {
      // Conflict detected - send back to client for resolution
      socket.emit("sync_conflict", {
        collection,
        id,
        local: changes,
        remote: existing,
        localTimestamp: timestamp,
        remoteTimestamp: existing._lastModified,
      });
      return;
    }

    // Apply change to database
    const updatedItem = {
      ...existing,
      ...changes,
      _lastModified: timestamp,
      _deviceId: deviceId,
      userId,
    };

    await db
      .collection(collection)
      .replaceOne({ id, userId }, updatedItem, { upsert: true });

    // Broadcast to other devices for this user
    socket.to(`user:${userId}`).emit("data_update", {
      collection,
      id,
      data: updatedItem,
      timestamp,
      deviceId,
      changeId: change.changeId,
    });
  }
}

// ❌ Blocking operations kill user experience
app.post("/upload-video", upload.single("video"), async (req, res) => {
  try {
    const videoFile = req.file;

    // These operations take 30+ seconds and block the request
    const processedVideo = await processVideo(videoFile); // 15 seconds
    const thumbnail = await generateThumbnail(videoFile); // 5 seconds
    const transcript = await generateTranscript(videoFile); // 20 seconds
    await sendNotificationToFollowers(req.user.id); // 5 seconds
    await updateRecommendationSystem(req.user.id); // 10 seconds

    res.json({ success: true, video: processedVideo });

    // User waited 55 seconds for response!
    // Server can't handle concurrent uploads
    // Memory usage spikes with large files
    // If any step fails, entire request fails
  } catch (error) {
    // User waited 30+ seconds just to see an error
    res.status(500).json({ error: "Upload failed" });
  }
});

// ✅ Instant response with reliable background processing
const Bull = require("bull");
const Redis = require("redis");

class JobManager {
  constructor() {
    this.redis = Redis.createClient(process.env.REDIS_URL);

    // Create specialized queues for different job types
    this.queues = {
      video: new Bull("video processing", { redis: this.redis }),
      notifications: new Bull("notifications", { redis: this.redis }),
      emails: new Bull("email sending", { redis: this.redis }),
      analytics: new Bull("analytics", { redis: this.redis }),
      ml: new Bull("machine learning", { redis: this.redis }),
    };

    this.setupWorkers();
    this.setupFailureHandling();
    this.setupMonitoring();
  }

  setupWorkers() {
    // Video processing worker - CPU intensive
    this.queues.video.process("process-video", 2, async (job) => {
      return await this.processVideoJob(job);
    });

    this.queues.video.process("generate-thumbnail", 5, async (job) => {
      return await this.generateThumbnailJob(job);
    });

    // Notification worker - I/O intensive
    this.queues.notifications.process("send-notification", 10, async (job) => {
      return await this.sendNotificationJob(job);
    });

    // Email worker - External API calls
    this.queues.emails.process("send-email", 5, async (job) => {
      return await this.sendEmailJob(job);
    });

    // Analytics worker - Database heavy
    this.queues.analytics.process("update-analytics", 3, async (job) => {
      return await this.updateAnalyticsJob(job);
    });

    // ML worker - Memory intensive
    this.queues.ml.process("train-model", 1, async (job) => {
      return await this.trainModelJob(job);
    });
  }

  async processVideoJob(job) {
    const { videoPath, userId, videoId } = job.data;

    try {
      // Update job progress
      job.progress(10);

      // Process video in multiple formats
      const formats = ["360p", "720p", "1080p"];
      const processedVideos = {};

      for (let i = 0; i < formats.length; i++) {
        const format = formats[i];
        job.progress(20 + i * 20);

        processedVideos[format] = await this.convertVideo(videoPath, format);
      }

      job.progress(80);

      // Update database
      await db.videos.updateOne(
        { id: videoId },
        {
          $set: {
            processedVideos,
            status: "processed",
            processedAt: new Date(),
          },
        }
      );

      job.progress(90);

      // Queue related jobs
      await this.queues.video.add(
        "generate-thumbnail",
        {
          videoPath,
          videoId,
          userId,
        },
        {
          delay: 1000, // Start after 1 second
          attempts: 3,
        }
      );

      await this.queues.notifications.add("video-processed", {
        userId,
        videoId,
        title: "Your video has been processed",
      });

      job.progress(100);

      return { success: true, processedVideos };
    } catch (error) {
      // Update database with error status
      await db.videos.updateOne(
        { id: videoId },
        {
          $set: {
            status: "failed",
            error: error.message,
            failedAt: new Date(),
          },
        }
      );

      throw error;
    }
  }

  async sendNotificationJob(job) {
    const { type, userId, data } = job.data;

    // Get user's notification preferences
    const user = await db.users.findOne({ id: userId });
    if (!user || !user.notifications?.enabled) {
      return { skipped: true, reason: "notifications_disabled" };
    }

    const notifications = [];

    // Send push notification if enabled
    if (user.notifications.push) {
      notifications.push(
        this.sendPushNotification(userId, {
          title: data.title,
          body: data.message,
          icon: "/icon-192x192.png",
          data: data.actionData,
        })
      );
    }

    // Send email notification if enabled
    if (user.notifications.email && this.shouldSendEmail(type, user)) {
      notifications.push(
        this.queues.emails.add("notification-email", {
          to: user.email,
          template: `notification-${type}`,
          data,
        })
      );
    }

    // Send in-app notification
    notifications.push(
      this.sendInAppNotification(userId, {
        type,
        title: data.title,
        message: data.message,
        createdAt: new Date(),
        read: false,
        actionUrl: data.actionUrl,
      })
    );

    await Promise.all(notifications);

    return { sent: notifications.length };
  }

  async scheduleRecurringJob(jobName, data, cronSchedule) {
    // Schedule recurring jobs (daily reports, weekly summaries, etc.)
    const job = await this.queues.analytics.add(jobName, data, {
      repeat: { cron: cronSchedule },
      removeOnComplete: 5, // Keep only 5 completed jobs
      removeOnFail: 10, // Keep 10 failed jobs for debugging
    });

    return job;
  }

  setupFailureHandling() {
    Object.values(this.queues).forEach((queue) => {
      queue.on("failed", async (job, error) => {
        console.error(`Job ${job.id} failed:`, error);

        // Implement exponential backoff for retries
        const retryCount = job.attemptsMade;
        const maxRetries = job.opts.attempts || 3;

        if (retryCount >= maxRetries) {
          // Job permanently failed
          await this.handleJobPermanentFailure(job, error);
        }

        // Log failure for monitoring
        await this.logJobFailure(job, error);
      });

      queue.on("completed", async (job, result) => {
        console.log(`Job ${job.id} completed successfully`);

        // Clean up temporary files if any
        await this.cleanupJobData(job);

        // Log success for monitoring
        await this.logJobSuccess(job, result);
      });
    });
  }

  async handleJobPermanentFailure(job, error) {
    const { userId, type } = job.data;

    // Notify user of failure if appropriate
    if (userId && this.shouldNotifyOfFailure(type)) {
      await this.queues.notifications.add("job-failed", {
        userId,
        jobType: job.name,
        error: error.message,
        supportContact: process.env.SUPPORT_EMAIL,
      });
    }

    // Send alert to development team
    await this.sendDeveloperAlert({
      jobId: job.id,
      jobName: job.name,
      error: error.message,
      data: job.data,
      attempts: job.attemptsMade,
      failedAt: new Date(),
    });
  }

  setupMonitoring() {
    // Health check endpoint
    setInterval(async () => {
      const stats = {
        timestamp: new Date(),
        queues: {},
      };

      for (const [name, queue] of Object.entries(this.queues)) {
        const waiting = await queue.getWaiting();
        const active = await queue.getActive();
        const completed = await queue.getCompleted();
        const failed = await queue.getFailed();

        stats.queues[name] = {
          waiting: waiting.length,
          active: active.length,
          completed: completed.length,
          failed: failed.length,
          health:
            failed.length / Math.max(completed.length + failed.length, 1) < 0.1
              ? "good"
              : "poor",
        };
      }

      // Send to monitoring service
      await this.sendToMonitoring(stats);

      // Alert if any queue is unhealthy
      Object.entries(stats.queues).forEach(([name, queueStats]) => {
        if (queueStats.waiting > 1000) {
          console.warn(
            `Queue ${name} has ${queueStats.waiting} waiting jobs - consider scaling workers`
          );
        }

        if (queueStats.health === "poor") {
          console.error(
            `Queue ${name} health is poor - ${queueStats.failed} failures detected`
          );
        }
      });
    }, 60000); // Check every minute
  }
}

// Usage in controllers
const jobManager = new JobManager();

app.post("/upload-video", upload.single("video"), async (req, res) => {
  try {
    const videoFile = req.file;
    const userId = req.user.id;
    const videoId = generateId();

    // Save basic video record immediately
    const video = {
      id: videoId,
      userId,
      title: req.body.title,
      originalFilename: videoFile.originalname,
      size: videoFile.size,
      status: "processing",
      createdAt: new Date(),
    };

    await db.videos.insertOne(video);

    // Save file to temporary location
    const tempPath = `/tmp/videos/${videoId}-${videoFile.originalname}`;
    await fs.writeFile(tempPath, videoFile.buffer);

    // Queue video processing job
    const processingJob = await jobManager.queues.video.add(
      "process-video",
      {
        videoPath: tempPath,
        userId,
        videoId,
      },
      {
        attempts: 3,
        backoff: {
          type: "exponential",
          delay: 5000, // Start with 5 second delay
        },
        removeOnComplete: 10,
        removeOnFail: 5,
      }
    );

    // Return immediately with job ID for progress tracking
    res.json({
      success: true,
      video: {
        id: videoId,
        status: "processing",
        jobId: processingJob.id,
      },
      message: "Video uploaded successfully and is being processed",
    });

    // User gets instant response!
  } catch (error) {
    console.error("Video upload error:", error);
    res.status(500).json({ error: "Upload failed" });
  }
});

// Job progress tracking endpoint
app.get("/jobs/:jobId/progress", async (req, res) => {
  try {
    const { jobId } = req.params;

    // Find job in all queues
    let job = null;
    for (const queue of Object.values(jobManager.queues)) {
      job = await queue.getJob(jobId);
      if (job) break;
    }

    if (!job) {
      return res.status(404).json({ error: "Job not found" });
    }

    const progress = {
      id: job.id,
      name: job.name,
      progress: job.progress(),
      status: await job.getState(),
      createdAt: new Date(job.timestamp),
      processedAt: job.processedOn ? new Date(job.processedOn) : null,
      finishedAt: job.finishedOn ? new Date(job.finishedOn) : null,
      attempts: job.attemptsMade,
      failedReason: job.failedReason,
    };

    res.json(progress);
  } catch (error) {
    console.error("Job progress error:", error);
    res.status(500).json({ error: "Failed to get job progress" });
  }
});

// Event-driven microservices communication
const EventEmitter = require("events");
const Redis = require("redis");

class EventBus extends EventEmitter {
  constructor() {
    super();
    this.redis = Redis.createClient(process.env.REDIS_URL);
    this.subscriber = this.redis.duplicate();
    this.setupSubscriptions();
    this.eventHistory = new Map(); // For event sourcing
  }

  async publish(eventType, data, options = {}) {
    const event = {
      id: generateId(),
      type: eventType,
      data,
      timestamp: Date.now(),
      source: options.source || "unknown",
      correlationId: options.correlationId,
      userId: options.userId,
      version: options.version || 1,
    };

    // Store in event history for replay/debugging
    this.eventHistory.set(event.id, event);

    // Publish locally
    this.emit(eventType, event);

    // Publish to Redis for cross-service communication
    await this.redis.publish("events", JSON.stringify(event));

    // Store in persistent event store
    await db.events.insertOne(event);

    console.log(`Published event: ${eventType}`, {
      id: event.id,
      source: event.source,
    });

    return event.id;
  }

  subscribe(eventType, handler, options = {}) {
    // Local subscription
    this.on(eventType, handler);

    // Store subscription metadata for monitoring
    if (!this.subscriptions) this.subscriptions = new Map();
    if (!this.subscriptions.has(eventType)) {
      this.subscriptions.set(eventType, []);
    }

    this.subscriptions.get(eventType).push({
      handler: handler.name || "anonymous",
      service: options.service || "unknown",
      subscribedAt: new Date(),
    });
  }

  setupSubscriptions() {
    this.subscriber.subscribe("events");
    this.subscriber.on("message", (channel, message) => {
      if (channel === "events") {
        const event = JSON.parse(message);
        // Emit locally for subscribers
        this.emit(event.type, event);
      }
    });
  }

  // Event replay for debugging or recovery
  async replayEvents(fromTimestamp, toTimestamp, eventTypes = []) {
    const query = {
      timestamp: {
        $gte: fromTimestamp,
        $lte: toTimestamp,
      },
    };

    if (eventTypes.length > 0) {
      query.type = { $in: eventTypes };
    }

    const events = await db.events.find(query).sort({ timestamp: 1 }).toArray();

    console.log(`Replaying ${events.length} events`);

    for (const event of events) {
      this.emit(event.type, event);
      // Small delay to prevent overwhelming
      await new Promise((resolve) => setTimeout(resolve, 10));
    }

    return events.length;
  }
}

// Global event bus
const eventBus = new EventBus();

// User service events
class UserEvents {
  static async userRegistered(userData) {
    return await eventBus.publish(
      "user.registered",
      {
        userId: userData.id,
        email: userData.email,
        username: userData.username,
        registrationMethod: userData.method,
        referrer: userData.referrer,
      },
      {
        source: "user-service",
        userId: userData.id,
      }
    );
  }

  static async userProfileUpdated(userId, changes) {
    return await eventBus.publish(
      "user.profile.updated",
      {
        userId,
        changes,
        updatedFields: Object.keys(changes),
      },
      {
        source: "user-service",
        userId,
      }
    );
  }

  static async userDeactivated(userId, reason) {
    return await eventBus.publish(
      "user.deactivated",
      {
        userId,
        reason,
        deactivatedAt: new Date(),
      },
      {
        source: "user-service",
        userId,
      }
    );
  }
}

// Content service event handlers
class ContentEventHandlers {
  static setupHandlers() {
    // When user registers, create default content preferences
    eventBus.subscribe(
      "user.registered",
      async (event) => {
        const { userId, email } = event.data;

        try {
          await db.contentPreferences.insertOne({
            userId,
            preferences: {
              topics: ["technology", "programming"],
              difficulty: "beginner",
              formats: ["article", "video"],
              notifications: true,
            },
            createdAt: new Date(),
          });

          // Start onboarding sequence
          await eventBus.publish(
            "content.onboarding.started",
            {
              userId,
              email,
            },
            {
              source: "content-service",
              userId,
              correlationId: event.correlationId,
            }
          );
        } catch (error) {
          console.error("Failed to setup content preferences:", error);

          // Publish error event for monitoring
          await eventBus.publish(
            "content.setup.failed",
            {
              userId,
              error: error.message,
              originalEvent: event.id,
            },
            {
              source: "content-service",
            }
          );
        }
      },
      { service: "content-service" }
    );

    // When user profile updates, adjust content recommendations
    eventBus.subscribe(
      "user.profile.updated",
      async (event) => {
        const { userId, changes } = event.data;

        if (changes.interests || changes.skillLevel || changes.preferences) {
          try {
            await jobManager.queues.ml.add(
              "update-recommendations",
              {
                userId,
                trigger: "profile_update",
                changes,
              },
              {
                delay: 5000, // Wait 5 seconds for other updates
                attempts: 2,
              }
            );
          } catch (error) {
            console.error("Failed to queue recommendation update:", error);
          }
        }
      },
      { service: "content-service" }
    );

    // When user deactivates, clean up content
    eventBus.subscribe(
      "user.deactivated",
      async (event) => {
        const { userId } = event.data;

        // Queue cleanup jobs
        await Promise.all([
          jobManager.queues.analytics.add("cleanup-user-data", {
            userId,
            dataTypes: [
              "content_preferences",
              "reading_history",
              "recommendations",
            ],
          }),

          eventBus.publish(
            "content.user.cleanup",
            {
              userId,
            },
            {
              source: "content-service",
              correlationId: event.correlationId,
            }
          ),
        ]);
      },
      { service: "content-service" }
    );
  }
}

// Analytics event handlers
class AnalyticsEventHandlers {
  static setupHandlers() {
    // Track all user events for analytics
    const userEvents = [
      "user.registered",
      "user.login",
      "user.logout",
      "user.profile.updated",
    ];

    userEvents.forEach((eventType) => {
      eventBus.subscribe(
        eventType,
        async (event) => {
          try {
            await db.analytics.insertOne({
              eventId: event.id,
              eventType: event.type,
              userId: event.data.userId,
              timestamp: event.timestamp,
              data: event.data,
              source: event.source,
            });

            // Update real-time metrics
            await this.updateRealTimeMetrics(event);
          } catch (error) {
            console.error("Analytics tracking failed:", error);
          }
        },
        { service: "analytics-service" }
      );
    });
  }

  static async updateRealTimeMetrics(event) {
    const today = new Date().toISOString().split("T")[0];
    const metricKey = `metrics:${today}:${event.type}`;

    // Increment daily counter
    await eventBus.redis.incr(metricKey);
    await eventBus.redis.expire(metricKey, 86400 * 7); // Keep for 7 days

    // Update hourly metrics
    const hour = new Date().getHours();
    const hourlyKey = `metrics:${today}:${hour}:${event.type}`;
    await eventBus.redis.incr(hourlyKey);
    await eventBus.redis.expire(hourlyKey, 86400); // Keep for 1 day
  }
}

// Email service event handlers
class EmailEventHandlers {
  static setupHandlers() {
    eventBus.subscribe("user.registered", async (event) => {
      const { userId, email, username } = event.data;

      // Send welcome email
      await jobManager.queues.emails.add(
        "welcome-email",
        {
          to: email,
          username,
          userId,
          template: "welcome",
          data: {
            username,
            verificationUrl: `${process.env.BASE_URL}/verify/${userId}`,
          },
        },
        {
          delay: 2000, // 2 second delay
          attempts: 3,
        }
      );
    });

    eventBus.subscribe("content.onboarding.started", async (event) => {
      const { userId, email } = event.data;

      // Send onboarding email sequence
      await jobManager.queues.emails.add(
        "onboarding-sequence",
        {
          userId,
          email,
          sequence: "new-user",
          step: 1,
        },
        {
          delay: 24 * 60 * 60 * 1000, // 24 hours delay
          attempts: 2,
        }
      );
    });
  }
}

// Initialize event handlers
ContentEventHandlers.setupHandlers();
AnalyticsEventHandlers.setupHandlers();
EmailEventHandlers.setupHandlers();