Tessl Skill Review CI/CD: CircleCI

version: 2.1

# Optional: use path-filtering orb to only run on skill changes
# orbs:
#   path-filtering: circleci/path-filtering@1.1.0

parameters:
  target-branch:
    type: string
    default: "main"
  cache-file:
    type: string
    default: ".tessl/skill-review-cache.json"
  block-on-failure:
    type: boolean
    default: true

jobs:
  review-skills:
    docker:
      - image: cimg/node:20.11
    resource_class: medium
    steps:
      - checkout

      - run:
          name: Install Tessl CLI
          command: npm install -g @tessl/cli

      - run:
          name: Detect changed skills
          command: |
            set -euo pipefail

            if [[ -n "${CIRCLE_PULL_REQUEST:-}" ]]; then
              # Extract PR number from URL
              PR_NUMBER=$(echo "$CIRCLE_PULL_REQUEST" | grep -oE '[0-9]+$')
              echo "Running in PR context (PR #${PR_NUMBER})"
              echo "$PR_NUMBER" > /tmp/pr_number

              # Detect changed SKILL.md files between PR branch and target
              TARGET_BRANCH="origin/<< pipeline.parameters.target-branch >>"
              git fetch origin << pipeline.parameters.target-branch >> --depth=50 || true

              CHANGED_SKILLS=$(git diff --name-only --diff-filter=ACMR \
                "${TARGET_BRANCH}"...HEAD \
                -- '**/SKILL.md' '**/skills/**' | \
                grep 'SKILL.md$' | \
                xargs -I {} dirname {} | \
                sort -u) || true
            else
              echo "Running on push to main or manual trigger"
              CHANGED_SKILLS=$(find . -name "SKILL.md" \
                -not -path "./node_modules/*" \
                -not -path "./.git/*" | \
                xargs -I {} dirname {} | \
                sed 's|^\./||' | \
                sort -u) || true
            fi

            if [[ -z "$CHANGED_SKILLS" ]]; then
              echo "No skill changes detected."
              echo "" > /tmp/changed_skills.txt
              circleci-agent step halt
            else
              echo "Skills to review:"
              echo "$CHANGED_SKILLS"
              echo "$CHANGED_SKILLS" > /tmp/changed_skills.txt
            fi

      - run:
          name: Read review cache
          command: |
            set -euo pipefail

            CACHE_FILE="<< pipeline.parameters.cache-file >>"

            if [[ -f "$CACHE_FILE" ]]; then
              echo "Cache file found, loading..."
              if cat "$CACHE_FILE" | jq empty 2>/dev/null; then
                echo "Cache is valid JSON"
                cp "$CACHE_FILE" /tmp/review_cache.json
              else
                echo "WARNING: Cache file is invalid JSON, ignoring"
                echo '{}' > /tmp/review_cache.json
              fi
            else
              echo "No cache file found, will create new one"
              echo '{}' > /tmp/review_cache.json
            fi

      - run:
          name: Run skill reviews
          command: |
            set -euo pipefail

            SKILLS=$(cat /tmp/changed_skills.txt)
            if [[ -z "$SKILLS" ]]; then
              echo "No skills to review."
              exit 0
            fi

            FAILED=0
            TABLE="| Skill | Status | Review Score | Change |"
            TABLE="${TABLE}\n|-------|--------|--------------|--------|"
            DETAILS=""

            # Load cache
            REVIEW_CACHE=""
            if [[ -f /tmp/review_cache.json ]]; then
              REVIEW_CACHE=$(cat /tmp/review_cache.json)
            fi

            # Create temporary file for cache entries
            CACHE_FILE_TEMP=$(mktemp)

            while IFS= read -r dir; do
              [[ -z "$dir" ]] && continue
              echo "======== Reviewing $dir ========"

              # Run review with --json flag
              JSON_OUTPUT=$(tessl skill review --json "$dir" 2>&1) || true
              echo "$JSON_OUTPUT"

              # Extract JSON (skip everything before first '{')
              JSON=$(echo "$JSON_OUTPUT" | sed -n '/{/,$p')

              # Look up previous score from cache
              PREV_SCORE=""
              PREV_DESC=""
              PREV_CONTENT=""
              if [[ -n "$REVIEW_CACHE" ]]; then
                CACHE_ENTRY=$(echo "$REVIEW_CACHE" | jq -r --arg path "$dir" '.skills[$path] // empty' 2>/dev/null) || true
                if [[ -n "$CACHE_ENTRY" ]]; then
                  PREV_SCORE=$(echo "$CACHE_ENTRY" | jq -r '.score // empty')
                  PREV_DESC=$(echo "$CACHE_ENTRY" | jq -r '.dimensions.description // empty')
                  PREV_CONTENT=$(echo "$CACHE_ENTRY" | jq -r '.dimensions.content // empty')
                fi
              fi

              # Validate previous scores are numeric
              if [[ -n "$PREV_SCORE" && ! "$PREV_SCORE" =~ ^[0-9]+$ ]]; then
                echo "WARNING: Invalid previous score for $dir: $PREV_SCORE, ignoring"
                PREV_SCORE=""
              fi
              if [[ -n "$PREV_DESC" && ! "$PREV_DESC" =~ ^[0-9]+$ ]]; then
                PREV_DESC=""
              fi
              if [[ -n "$PREV_CONTENT" && ! "$PREV_CONTENT" =~ ^[0-9]+$ ]]; then
                PREV_CONTENT=""
              fi

              # Extract validation status
              PASSED=$(echo "$JSON" | jq -r '.validation.overallPassed // false')

              # Calculate average score from all dimensions
              AVG_SCORE=$(echo "$JSON" | jq -r '
                def avg(obj): (obj.scores | to_entries | map(.value.score) | add) / (obj.scores | length) * 100 / 3;
                (
                  [(.descriptionJudge.evaluation | avg(.)), (.contentJudge.evaluation | avg(.))] | add / 2
                ) | round
              ')

              # Validate AVG_SCORE is numeric
              if [[ ! "$AVG_SCORE" =~ ^[0-9]+$ ]]; then
                echo "ERROR: Invalid average score for $dir: $AVG_SCORE"
                AVG_SCORE=0
              fi

              # Calculate diff against previous score
              CHANGE=""
              if [[ -n "$PREV_SCORE" ]]; then
                DIFF=$((AVG_SCORE - PREV_SCORE))
                if [[ $DIFF -gt 0 ]]; then
                  CHANGE="🔺 +${DIFF}% (was ${PREV_SCORE}%)"
                elif [[ $DIFF -lt 0 ]]; then
                  CHANGE="🔻 ${DIFF}% (was ${PREV_SCORE}%)"
                else
                  CHANGE="➡️ no change"
                fi
              fi

              # Build status column
              if [[ "$PASSED" == "true" ]]; then
                STATUS="✅ PASSED"
              else
                ERROR=$(echo "$JSON" | jq -r '
                  .validation.checks
                  | map(select(.status != "passed"))
                  | first
                  | .message // "Validation failed"
                ' | cut -c1-60)
                STATUS="❌ FAILED — ${ERROR}"
                FAILED=1
              fi

              DIR_DISPLAY=$(echo "$dir" | tr '|' '/')
              TABLE="${TABLE}\n| \`${DIR_DISPLAY}\` | ${STATUS} | ${AVG_SCORE}% | ${CHANGE} |"

              # Calculate dimension scores
              DESC_SCORE=$(echo "$JSON" | jq -r '
                (.descriptionJudge.evaluation.scores | to_entries | map(.value.score) | add) * 100 / ((.descriptionJudge.evaluation.scores | length) * 3) | round
              ')
              CONTENT_SCORE=$(echo "$JSON" | jq -r '
                (.contentJudge.evaluation.scores | to_entries | map(.value.score) | add) * 100 / ((.contentJudge.evaluation.scores | length) * 3) | round
              ')

              if [[ ! "$DESC_SCORE" =~ ^[0-9]+$ ]]; then DESC_SCORE=0; fi
              if [[ ! "$CONTENT_SCORE" =~ ^[0-9]+$ ]]; then CONTENT_SCORE=0; fi

              # Extract detailed evaluations
              DESC_EVAL=$(echo "$JSON" | jq -r '.descriptionJudge.evaluation |
                "  Description: " + ((.scores | to_entries | map(.value.score) | add) * 100 / ((.scores | length) * 3) | round | tostring) + "%\n" +
                (.scores | to_entries | map("    \(.key): \(.value.score)/3 - \(.value.reasoning)") | join("\n")) + "\n\n" +
                "    Assessment: " + .overall_assessment
              ')

              CONTENT_EVAL=$(echo "$JSON" | jq -r '.contentJudge.evaluation |
                "  Content: " + ((.scores | to_entries | map(.value.score) | add) * 100 / ((.scores | length) * 3) | round | tostring) + "%\n" +
                (.scores | to_entries | map("    \(.key): \(.value.score)/3 - \(.value.reasoning)") | join("\n")) + "\n\n" +
                "    Assessment: " + .overall_assessment
              ')

              SUGGESTIONS=$(echo "$JSON" | jq -r '
                [.descriptionJudge.evaluation.suggestions // [], .contentJudge.evaluation.suggestions // []]
                | flatten
                | map("- " + .)
                | join("\n")
              ')

              # Build collapsible details block (Markdown)
              DETAILS="${DETAILS}\n\n<details>\n<summary><strong>${DIR_DISPLAY}</strong> — ${AVG_SCORE}% (${STATUS#* })</summary>\n\n"

              if [[ -n "$PREV_SCORE" && -n "$PREV_DESC" && -n "$PREV_CONTENT" ]]; then
                DETAILS="${DETAILS}**Previous:** ${PREV_SCORE}% (Description: ${PREV_DESC}%, Content: ${PREV_CONTENT}%)\n"
                DETAILS="${DETAILS}**Current:**  ${AVG_SCORE}% (Description: ${DESC_SCORE}%, Content: ${CONTENT_SCORE}%)\n\n"
                DETAILS="${DETAILS}---\n\n"
              fi

              DETAILS="${DETAILS}\`\`\`\n${DESC_EVAL}\n\n${CONTENT_EVAL}\n\`\`\`\n"

              if [[ -n "$SUGGESTIONS" ]]; then
                DETAILS="${DETAILS}\n**Suggestions:**\n\n${SUGGESTIONS}\n"
              fi

              DETAILS="${DETAILS}\n</details>"

              # Calculate content hash for cache
              if [[ ! -f "$dir/SKILL.md" ]]; then
                echo "ERROR: SKILL.md not found for $dir"
                continue
              fi
              CONTENT_HASH="sha256:$(sha256sum "$dir/SKILL.md" | awk '{print $1}')"

              # Build cache entry
              CACHE_ENTRY=$(jq -nc \
                --arg score "$AVG_SCORE" \
                --arg passed "$PASSED" \
                --arg hash "$CONTENT_HASH" \
                --arg ts "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
                --arg desc "$DESC_SCORE" \
                --arg content "$CONTENT_SCORE" \
                '{
                  score: ($score | tonumber),
                  validation_passed: ($passed == "true"),
                  content_hash: $hash,
                  timestamp: $ts,
                  dimensions: {
                    description: ($desc | tonumber),
                    content: ($content | tonumber)
                  }
                }')

              printf '%s\t%s\n' "$dir" "$CACHE_ENTRY" >> "$CACHE_FILE_TEMP"

            done <<< "$SKILLS"

            # Save cache entries for update step
            cp "$CACHE_FILE_TEMP" /tmp/cache_entries.tsv

            # Build PR comment body
            COMMENT_BODY=$(printf '%b' "<!-- tessl-skill-review -->\n## Tessl Skill Review Results\n\n${TABLE}\n\n---\n\n### Detailed Review\n${DETAILS}\n\n---\n_Checks: frontmatter validity, required fields, body structure, examples, line count._\n_Review score is informational — not used for pass/fail gating._")

            # Save comment for PR posting step
            echo "$COMMENT_BODY" > /tmp/comment.md

            if [[ "$FAILED" -eq 1 ]]; then
              echo "One or more skills failed validation checks."
              if [[ "<< pipeline.parameters.block-on-failure >>" == "true" ]]; then
                echo "block-on-failure is enabled — failing the build."
                exit 1
              else
                echo "block-on-failure is disabled — continuing with warnings."
              fi
            fi

      - run:
          name: Update review cache
          when: always
          command: |
            set -euo pipefail

            CACHE_FILE="<< pipeline.parameters.cache-file >>"
            mkdir -p "$(dirname "$CACHE_FILE")"

            # Load existing cache or create new structure
            if [[ -f "$CACHE_FILE" ]]; then
              CACHE=$(cat "$CACHE_FILE")
              if ! echo "$CACHE" | jq empty 2>/dev/null; then
                echo "WARNING: Cache file is invalid JSON, recreating"
                CACHE='{"version":"1","last_updated":"","skills":{}}'
              fi
            else
              echo "Creating new cache file..."
              CACHE='{"version":"1","last_updated":"","skills":{}}'
            fi

            # Update timestamp
            TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
            CACHE=$(echo "$CACHE" | jq --arg ts "$TIMESTAMP" '.last_updated = $ts')

            # Merge cache updates
            if [[ -f /tmp/cache_entries.tsv ]]; then
              MERGED_COUNT=0
              FAILED_COUNT=0

              while IFS=$'\t' read -r skill_path entry_json; do
                [[ -z "$skill_path" ]] && continue
                if NEW_CACHE=$(echo "$CACHE" | jq --arg path "$skill_path" --argjson entry "$entry_json" \
                  '.skills[$path] = $entry' 2>&1); then
                  CACHE="$NEW_CACHE"
                  MERGED_COUNT=$((MERGED_COUNT + 1))
                else
                  echo "WARNING: Failed to merge cache entry for $skill_path: $NEW_CACHE"
                  FAILED_COUNT=$((FAILED_COUNT + 1))
                fi
              done < /tmp/cache_entries.tsv

              echo "Cache updated with $MERGED_COUNT entries ($FAILED_COUNT failed)"
            fi

            # Write updated cache
            echo "$CACHE" | jq '.' > "$CACHE_FILE"

            # Copy to workspace for downstream jobs
            mkdir -p /tmp/workspace
            cp "$CACHE_FILE" /tmp/workspace/skill-review-cache.json
            if [[ -f /tmp/comment.md ]]; then
              cp /tmp/comment.md /tmp/workspace/comment.md
            fi
            if [[ -f /tmp/pr_number ]]; then
              cp /tmp/pr_number /tmp/workspace/pr_number
            fi

      - persist_to_workspace:
          root: /tmp/workspace
          paths:
            - skill-review-cache.json
            - comment.md
            - pr_number

      - store_artifacts:
          path: /tmp/workspace/skill-review-cache.json
          destination: skill-review-cache.json

      - store_artifacts:
          path: /tmp/workspace/comment.md
          destination: comment.md

  post-pr-comment:
    docker:
      - image: cimg/base:current
    resource_class: small
    steps:
      - attach_workspace:
          at: /tmp/workspace

      - run:
          name: Post PR comment
          command: |
            set -euo pipefail

            if [[ ! -f /tmp/workspace/pr_number ]]; then
              echo "Not a PR build, skipping comment."
              exit 0
            fi

            if [[ ! -f /tmp/workspace/comment.md ]]; then
              echo "No comment file found, skipping."
              exit 0
            fi

            PR_NUMBER=$(cat /tmp/workspace/pr_number)
            COMMENT_BODY=$(cat /tmp/workspace/comment.md)

            # Extract owner/repo from CIRCLE_REPOSITORY_URL
            # Handles both https://github.com/owner/repo and git@github.com:owner/repo.git
            REPO_SLUG=$(echo "$CIRCLE_REPOSITORY_URL" | sed -E 's|.*github\.com[:/]||; s|\.git$||')

            echo "Posting comment to ${REPO_SLUG} PR #${PR_NUMBER}..."

            # Check for existing comment to update
            EXISTING_COMMENT_ID=$(curl -s \
              -H "Authorization: token ${GITHUB_TOKEN}" \
              -H "Accept: application/vnd.github.v3+json" \
              "https://api.github.com/repos/${REPO_SLUG}/issues/${PR_NUMBER}/comments" | \
              jq -r '.[] | select(.body | test("<!-- tessl-skill-review -->")) | .id' | head -1)

            # Escape body for JSON
            ESCAPED_BODY=$(jq -Rs '.' <<< "$COMMENT_BODY")

            if [[ -n "$EXISTING_COMMENT_ID" && "$EXISTING_COMMENT_ID" != "null" ]]; then
              echo "Updating existing comment (ID: $EXISTING_COMMENT_ID)..."
              curl -s -X PATCH \
                -H "Authorization: token ${GITHUB_TOKEN}" \
                -H "Accept: application/vnd.github.v3+json" \
                "https://api.github.com/repos/${REPO_SLUG}/issues/comments/${EXISTING_COMMENT_ID}" \
                -d "{\"body\": ${ESCAPED_BODY}}"
              echo "Comment updated."
            else
              echo "Creating new comment..."
              curl -s -X POST \
                -H "Authorization: token ${GITHUB_TOKEN}" \
                -H "Accept: application/vnd.github.v3+json" \
                "https://api.github.com/repos/${REPO_SLUG}/issues/${PR_NUMBER}/comments" \
                -d "{\"body\": ${ESCAPED_BODY}}"
              echo "Comment posted."
            fi

  commit-cache:
    docker:
      - image: cimg/base:current
    resource_class: small
    steps:
      - checkout

      - attach_workspace:
          at: /tmp/workspace

      - run:
          name: Commit cache update
          command: |
            set -euo pipefail

            CACHE_FILE="<< pipeline.parameters.cache-file >>"
            mkdir -p "$(dirname "$CACHE_FILE")"

            # Copy cache from workspace
            cp /tmp/workspace/skill-review-cache.json "$CACHE_FILE"

            # Check if cache actually changed
            if git diff --quiet HEAD -- "$CACHE_FILE" 2>/dev/null; then
              echo "Cache file unchanged, nothing to commit."
              exit 0
            fi

            echo "Cache file has changes, committing..."

            git config user.name "circleci[bot]"
            git config user.email "circleci[bot]@users.noreply.github.com"
            git add "$CACHE_FILE"
            git commit -m "chore: update skill review cache [skip ci]"

            # Push to target branch
            git push origin HEAD:<< pipeline.parameters.target-branch >>

            echo "Cache committed and pushed."

workflows:
  skill-review:
    jobs:
      - review-skills:
          context:
            - tessl-credentials
          filters:
            branches:
              ignore:
                - /^dependabot\/.*/

      - post-pr-comment:
          requires:
            - review-skills
          context:
            - tessl-credentials
          filters:
            branches:
              ignore:
                - << pipeline.parameters.target-branch >>
                - /^dependabot\/.*/

      - commit-cache:
          requires:
            - review-skills
          context:
            - tessl-credentials
          filters:
            branches:
              only:
                - << pipeline.parameters.target-branch >>

mkdir -p .circleci
# Copy the template above into .circleci/config.yml

mkdir -p .tessl
cat > .tessl/skill-review-cache.json << 'EOF'
{
  "version": "1",
  "last_updated": "",
  "skills": {}
}
EOF
git add .tessl/skill-review-cache.json
git commit -m "feat: initialize skill review cache"
git push

- run:
    name: Configure git auth
    command: |
      git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git

parameters:
  block-on-failure:
    type: boolean
    default: true

parameters:
  block-on-failure:
    type: boolean
    default: false

curl -X POST \
  -H "Circle-Token: ${CIRCLECI_TOKEN}" \
  -H "Content-Type: application/json" \
  -d '{"parameters": {"block-on-failure": false}}' \
  "https://circleci.com/api/v2/project/gh/${ORG}/${REPO}/pipeline"

# Bitbucket PR comment API
curl -s -X POST \
  -H "Authorization: Bearer ${BITBUCKET_TOKEN}" \
  -H "Content-Type: application/json" \
  "https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_SLUG}/pullrequests/${PR_ID}/comments" \
  -d "{\"content\": {\"raw\": ${ESCAPED_BODY}}}"

filters:
  branches:
    ignore:
      - /^dependabot\/.*/

1. Push the .circleci/config.yml to your repository
2. Go to CircleCI dashboard > select your project
3. Verify the pipeline runs on the push

Verify:
  - Pipeline runs without errors
  - Review scores appear in job output
  - Cache artifact is stored

1. Create a test branch:
   git checkout -b test/skill-review-setup

2. Modify a SKILL.md file:
   echo "Updated for testing" >> path/to/SKILL.md

3. Commit and push:
   git add path/to/SKILL.md
   git commit -m "test: trigger skill review pipeline"
   git push -u origin test/skill-review-setup

4. Create a Pull Request on GitHub

5. Wait for CircleCI pipeline to run

6. Verify:
   - review-skills job runs and shows scores
   - post-pr-comment job runs
   - PR comment appears with review results table
   - Detailed evaluations are in collapsible sections

1. Merge the test PR to main

2. Wait for the pipeline to run on the push to main

3. Verify:
   - commit-cache job runs successfully
   - New commit appears: "chore: update skill review cache [skip ci]"
   - .tessl/skill-review-cache.json contains skill entries
   - Pipeline does NOT trigger again from the cache commit

4. Create another PR modifying the same skill

5. Verify:
   - Score diff indicators appear (🔺 🔻 ➡️)
   - Previous vs Current scores are shown in details

1. Go to CircleCI dashboard > select your project > select pipeline
2. Click "Trigger Pipeline"
3. Select branch: main
4. Click "Trigger Pipeline"

Verify:
  - All skills are reviewed (not just changed ones)
  - No PR comment is posted (expected for non-PR runs)
  - Cache is updated

1. Create a PR with an intentionally broken SKILL.md (e.g., missing frontmatter)

With block-on-failure: true:
  - review-skills job fails
  - PR status check shows failure
  - PR cannot be merged (if CircleCI is a required check)

With block-on-failure: false:
  - review-skills job succeeds (with warnings in output)
  - PR comment still shows the failure details
  - PR status check shows success