Building Data Pipelines with dlt MCP and Continue

name: Data Pipeline Validation with dlt MCP

on:
  pull_request:
    branches: [main]
  workflow_dispatch:

jobs:
  validate-pipeline:
    runs-on: ubuntu-latest
    env:
      CONTINUE_API_KEY: ${{ secrets.CONTINUE_API_KEY }}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "18"

      - name: Install dlt
        run: |
          pip install dlt
          echo "✅ dlt installed"

      - name: Install Continue CLI
        run: |
          npm install -g @continuedev/cli
          echo "✅ Continue CLI installed"

      - name: Validate Pipeline Schema
        run: |
          echo "🔍 Validating pipeline schema..."
          cn --agent continuedev/dlt-agent \
             -p "Inspect the pipeline schema and verify all required tables
                 and columns are present. Flag any missing or unexpected changes." \
             --auto

      - name: Check Pipeline Health
        run: |
          echo "📊 Checking pipeline health..."
          cn --agent continuedev/dlt-agent \
             -p "Analyze the last pipeline run for errors or warnings.
                 Report any issues that need attention." \
             --auto

      - name: Comment Pipeline Report on PR
        if: always() && github.event_name == 'pull_request'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          REPORT=$(cn --agent continuedev/dlt-agent \
             -p "Generate a concise summary (200 words or less) of:
                 - Pipeline schemas and row counts
                 - Any load errors or warnings
                 - Performance metrics (timing, file sizes)
                 - Recommended improvements" \
             --auto)

          gh pr comment ${{ github.event.pull_request.number }} --body "$REPORT"

"Check if there's a dlt pipeline in the current directory.
If not, help me initialize a new pipeline."

"Verify the destination connection and credentials for my pipeline.
Test the connection and report any issues."