n8n-mcp/src/community/community-node-fetcher.ts

import axios, { AxiosError } from 'axios';
import { logger } from '../utils/logger';

/**
 * Configuration constants for community node fetching
 */
const FETCH_CONFIG = {
  /** Default timeout for Strapi API requests (ms) */
  STRAPI_TIMEOUT: 30000,
  /** Default timeout for npm registry requests (ms) */
  NPM_REGISTRY_TIMEOUT: 15000,
  /** Default timeout for npm downloads API (ms) */
  NPM_DOWNLOADS_TIMEOUT: 10000,
  /** Base delay between retries (ms) */
  RETRY_DELAY: 1000,
  /** Maximum number of retry attempts */
  MAX_RETRIES: 3,
  /** Default delay between requests for rate limiting (ms) */
  RATE_LIMIT_DELAY: 300,
  /** Default delay after hitting 429 (ms) */
  RATE_LIMIT_429_DELAY: 60000,
} as const;

/**
 * Strapi API response types for verified community nodes
 */
export interface StrapiCommunityNodeAttributes {
  name: string;
  displayName: string;
  description: string;
  packageName: string;
  authorName: string;
  authorGithubUrl?: string;
  npmVersion: string;
  numberOfDownloads: number;
  numberOfStars: number;
  isOfficialNode: boolean;
  isPublished: boolean;
  nodeDescription: any; // Complete n8n node schema
  nodeVersions?: any[];
  checksum?: string;
  createdAt: string;
  updatedAt: string;
}

export interface StrapiCommunityNode {
  id: number;
  attributes: StrapiCommunityNodeAttributes;
}

export interface StrapiPaginatedResponse<T> {
  data: Array<{ id: number; attributes: T }>;
  meta: {
    pagination: {
      page: number;
      pageSize: number;
      pageCount: number;
      total: number;
    };
  };
}

/**
 * npm registry search response types
 */
export interface NpmPackageInfo {
  name: string;
  version: string;
  description: string;
  keywords: string[];
  date: string;
  links: {
    npm: string;
    homepage?: string;
    repository?: string;
  };
  author?: {
    name?: string;
    email?: string;
    username?: string;
  };
  publisher?: {
    username: string;
    email: string;
  };
  maintainers: Array<{ username: string; email: string }>;
}

export interface NpmSearchResult {
  package: NpmPackageInfo;
  score: {
    final: number;
    detail: {
      quality: number;
      popularity: number;
      maintenance: number;
    };
  };
  searchScore: number;
}

export interface NpmSearchResponse {
  objects: NpmSearchResult[];
  total: number;
  time: string;
}

/**
 * Response type for full package data including README
 */
export interface NpmPackageWithReadme {
  name: string;
  version: string;
  description?: string;
  readme?: string;
  readmeFilename?: string;
  homepage?: string;
  repository?: {
    type?: string;
    url?: string;
  };
  keywords?: string[];
  license?: string;
  'dist-tags'?: {
    latest?: string;
  };
}

/**
 * Fetches community nodes from n8n Strapi API and npm registry.
 * Follows the pattern from template-fetcher.ts.
 */
export class CommunityNodeFetcher {
  private readonly strapiBaseUrl: string;
  private readonly npmSearchUrl = 'https://registry.npmjs.org/-/v1/search';
  private readonly npmRegistryUrl = 'https://registry.npmjs.org';
  private readonly maxRetries = FETCH_CONFIG.MAX_RETRIES;
  private readonly retryDelay = FETCH_CONFIG.RETRY_DELAY;
  private readonly strapiPageSize = 25;
  private readonly npmPageSize = 250; // npm API max

  /** Regex for validating npm package names per npm naming rules */
  private readonly npmPackageNameRegex = /^(@[a-z0-9-~][a-z0-9-._~]*\/)?[a-z0-9-~][a-z0-9-._~]*$/;

  constructor(environment: 'production' | 'staging' = 'production') {
    this.strapiBaseUrl =
      environment === 'production'
        ? 'https://api.n8n.io/api/community-nodes'
        : 'https://api-staging.n8n.io/api/community-nodes';
  }

  /**
   * Validates npm package name to prevent path traversal and injection attacks.
   * @see https://github.com/npm/validate-npm-package-name
   */
  private validatePackageName(packageName: string): boolean {
    if (!packageName || typeof packageName !== 'string') {
      return false;
    }
    // Max length per npm spec
    if (packageName.length > 214) {
      return false;
    }
    // Must match npm naming pattern
    if (!this.npmPackageNameRegex.test(packageName)) {
      return false;
    }
    // Block path traversal attempts
    if (packageName.includes('..') || packageName.includes('//')) {
      return false;
    }
    return true;
  }

  /**
   * Checks if an error is a rate limit (429) response
   */
  private isRateLimitError(error: unknown): boolean {
    return axios.isAxiosError(error) && error.response?.status === 429;
  }

  /**
   * Retry helper for API calls (same pattern as TemplateFetcher)
   * Handles 429 rate limit responses with extended delay
   */
  private async retryWithBackoff<T>(
    fn: () => Promise<T>,
    context: string,
    maxRetries: number = this.maxRetries
  ): Promise<T | null> {
    let lastError: unknown;

    for (let attempt = 1; attempt <= maxRetries; attempt++) {
      try {
        return await fn();
      } catch (error: unknown) {
        lastError = error;

        if (attempt < maxRetries) {
          // Handle 429 rate limit with longer delay
          if (this.isRateLimitError(error)) {
            const delay = FETCH_CONFIG.RATE_LIMIT_429_DELAY;
            logger.warn(
              `${context} - Rate limited (429), waiting ${delay / 1000}s before retry...`
            );
            await this.sleep(delay);
          } else {
            const delay = this.retryDelay * attempt; // Exponential backoff
            logger.warn(
              `${context} - Attempt ${attempt}/${maxRetries} failed, retrying in ${delay}ms...`
            );
            await this.sleep(delay);
          }
        }
      }
    }

    logger.error(`${context} - All ${maxRetries} attempts failed, skipping`, lastError);
    return null;
  }

  /**
   * Fetch all verified community nodes from n8n Strapi API.
   * These nodes include full nodeDescription schemas - no parsing needed!
   */
  async fetchVerifiedNodes(
    progressCallback?: (message: string, current: number, total: number) => void
  ): Promise<StrapiCommunityNode[]> {
    const allNodes: StrapiCommunityNode[] = [];
    let page = 1;
    let hasMore = true;
    let total = 0;

    logger.info('Fetching verified community nodes from n8n Strapi API...');

    while (hasMore) {
      const result = await this.retryWithBackoff(
        async () => {
          const response = await axios.get<StrapiPaginatedResponse<StrapiCommunityNodeAttributes>>(
            this.strapiBaseUrl,
            {
              params: {
                'pagination[page]': page,
                'pagination[pageSize]': this.strapiPageSize,
              },
              timeout: FETCH_CONFIG.STRAPI_TIMEOUT,
            }
          );
          return response.data;
        },
        `Fetching verified nodes page ${page}`
      );

      if (result === null) {
        logger.warn(`Skipping page ${page} after failed attempts`);
        page++;
        continue;
      }

      const nodes = result.data.map((item) => ({
        id: item.id,
        attributes: item.attributes,
      }));

      allNodes.push(...nodes);
      total = result.meta.pagination.total;

      if (progressCallback) {
        progressCallback(`Fetching verified nodes`, allNodes.length, total);
      }

      logger.debug(
        `Fetched page ${page}/${result.meta.pagination.pageCount}: ${nodes.length} nodes (total: ${allNodes.length}/${total})`
      );

      // Check if there are more pages
      if (page >= result.meta.pagination.pageCount) {
        hasMore = false;
      }

      page++;

      // Rate limiting
      if (hasMore) {
        await this.sleep(FETCH_CONFIG.RATE_LIMIT_DELAY);
      }
    }

    logger.info(`Fetched ${allNodes.length} verified community nodes from Strapi API`);
    return allNodes;
  }

  /**
   * Fetch popular community node packages from npm registry.
   * Sorted by popularity (downloads). Returns package metadata only.
   * To get node schemas, packages need to be downloaded and parsed.
   *
   * @param limit Maximum number of packages to fetch (default: 100)
   */
  async fetchNpmPackages(
    limit: number = 100,
    progressCallback?: (message: string, current: number, total: number) => void
  ): Promise<NpmSearchResult[]> {
    const allPackages: NpmSearchResult[] = [];
    let offset = 0;
    const targetLimit = Math.min(limit, 1000); // npm API practical limit

    logger.info(`Fetching top ${targetLimit} community node packages from npm registry...`);

    while (allPackages.length < targetLimit) {
      const remaining = targetLimit - allPackages.length;
      const size = Math.min(this.npmPageSize, remaining);

      const result = await this.retryWithBackoff(
        async () => {
          const response = await axios.get<NpmSearchResponse>(this.npmSearchUrl, {
            params: {
              text: 'keywords:n8n-community-node-package',
              size,
              from: offset,
              // Sort by popularity (downloads)
              quality: 0,
              popularity: 1,
              maintenance: 0,
            },
            timeout: FETCH_CONFIG.STRAPI_TIMEOUT,
          });
          return response.data;
        },
        `Fetching npm packages (offset ${offset})`
      );

      if (result === null) {
        logger.warn(`Skipping npm fetch at offset ${offset} after failed attempts`);
        break;
      }

      if (result.objects.length === 0) {
        break; // No more packages
      }

      allPackages.push(...result.objects);

      if (progressCallback) {
        progressCallback(`Fetching npm packages`, allPackages.length, Math.min(result.total, targetLimit));
      }

      logger.debug(
        `Fetched ${result.objects.length} packages (total: ${allPackages.length}/${Math.min(result.total, targetLimit)})`
      );

      offset += size;

      // Rate limiting
      await this.sleep(FETCH_CONFIG.RATE_LIMIT_DELAY);
    }

    // Sort by popularity score (highest first)
    allPackages.sort((a, b) => b.score.detail.popularity - a.score.detail.popularity);

    logger.info(`Fetched ${allPackages.length} community node packages from npm`);
    return allPackages.slice(0, limit);
  }

  /**
   * Fetch package.json for a specific npm package to get the n8n node configuration.
   * Validates package name to prevent path traversal attacks.
   */
  async fetchPackageJson(packageName: string, version?: string): Promise<any | null> {
    // Validate package name to prevent path traversal
    if (!this.validatePackageName(packageName)) {
      logger.warn(`Invalid package name rejected: ${packageName}`);
      return null;
    }

    const url = version
      ? `${this.npmRegistryUrl}/${encodeURIComponent(packageName)}/${encodeURIComponent(version)}`
      : `${this.npmRegistryUrl}/${encodeURIComponent(packageName)}/latest`;

    return this.retryWithBackoff(
      async () => {
        const response = await axios.get(url, { timeout: FETCH_CONFIG.NPM_REGISTRY_TIMEOUT });
        return response.data;
      },
      `Fetching package.json for ${packageName}${version ? `@${version}` : ''}`
    );
  }

  /**
   * Download package tarball URL for a specific package version.
   * Returns the tarball URL that can be used to download and extract the package.
   */
  async getPackageTarballUrl(packageName: string, version?: string): Promise<string | null> {
    const packageJson = await this.fetchPackageJson(packageName, version);

    if (!packageJson) {
      return null;
    }

    // For specific version fetch, dist.tarball is directly available
    if (packageJson.dist?.tarball) {
      return packageJson.dist.tarball;
    }

    // For full package fetch, get the latest version's tarball
    const latestVersion = packageJson['dist-tags']?.latest;
    if (latestVersion && packageJson.versions?.[latestVersion]?.dist?.tarball) {
      return packageJson.versions[latestVersion].dist.tarball;
    }

    return null;
  }

  /**
   * Fetch full package data including README from npm registry.
   * Uses the base package URL (not /latest) to get the README field.
   * Validates package name to prevent path traversal attacks.
   *
   * @param packageName npm package name (e.g., "n8n-nodes-brightdata")
   * @returns Full package data including readme, or null if fetch failed
   */
  async fetchPackageWithReadme(packageName: string): Promise<NpmPackageWithReadme | null> {
    // Validate package name to prevent path traversal
    if (!this.validatePackageName(packageName)) {
      logger.warn(`Invalid package name rejected for README fetch: ${packageName}`);
      return null;
    }

    const url = `${this.npmRegistryUrl}/${encodeURIComponent(packageName)}`;

    return this.retryWithBackoff(
      async () => {
        const response = await axios.get<NpmPackageWithReadme>(url, {
          timeout: FETCH_CONFIG.NPM_REGISTRY_TIMEOUT,
        });
        return response.data;
      },
      `Fetching package with README for ${packageName}`
    );
  }

  /**
   * Fetch READMEs for multiple packages in batch with rate limiting.
   * Returns a Map of packageName -> readme content.
   *
   * @param packageNames Array of npm package names
   * @param progressCallback Optional callback for progress updates
   * @param concurrency Number of concurrent requests (default: 1 for rate limiting)
   * @returns Map of packageName to README content (null if not found)
   */
  async fetchReadmesBatch(
    packageNames: string[],
    progressCallback?: (message: string, current: number, total: number) => void,
    concurrency: number = 1
  ): Promise<Map<string, string | null>> {
    const results = new Map<string, string | null>();
    const total = packageNames.length;

    logger.info(`Fetching READMEs for ${total} packages (concurrency: ${concurrency})...`);

    // Process in batches based on concurrency
    for (let i = 0; i < packageNames.length; i += concurrency) {
      const batch = packageNames.slice(i, i + concurrency);

      // Process batch concurrently
      const batchPromises = batch.map(async (packageName) => {
        const data = await this.fetchPackageWithReadme(packageName);
        return { packageName, readme: data?.readme || null };
      });

      const batchResults = await Promise.all(batchPromises);

      for (const { packageName, readme } of batchResults) {
        results.set(packageName, readme);
      }

      if (progressCallback) {
        progressCallback('Fetching READMEs', Math.min(i + concurrency, total), total);
      }

      // Rate limiting between batches
      if (i + concurrency < packageNames.length) {
        await this.sleep(FETCH_CONFIG.RATE_LIMIT_DELAY);
      }
    }

    const foundCount = Array.from(results.values()).filter((v) => v !== null).length;
    logger.info(`Fetched ${foundCount}/${total} READMEs successfully`);

    return results;
  }

  /**
   * Get download statistics for a package from npm.
   * Validates package name to prevent path traversal attacks.
   */
  async getPackageDownloads(
    packageName: string,
    period: 'last-week' | 'last-month' = 'last-week'
  ): Promise<number | null> {
    // Validate package name to prevent path traversal
    if (!this.validatePackageName(packageName)) {
      logger.warn(`Invalid package name rejected for downloads: ${packageName}`);
      return null;
    }

    return this.retryWithBackoff(
      async () => {
        const response = await axios.get(
          `https://api.npmjs.org/downloads/point/${period}/${encodeURIComponent(packageName)}`,
          { timeout: FETCH_CONFIG.NPM_DOWNLOADS_TIMEOUT }
        );
        return response.data.downloads;
      },
      `Fetching downloads for ${packageName}`
    );
  }

  private sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms));
  }
}