Update .github/workflows/sync-content-to-repo.yml

.github/workflows/sync-content-to-repo.yml

@@ -0,0 +1,67 @@
+name: Sync Content to Repo
+
+on:
+  workflow_dispatch:
+    inputs:
+      roadmap_slug:
+        description: "The ID of the roadmap to sync"
+        required: true
+        default: "__default__"
+
+jobs:
+  sync-content:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup pnpm@v9
+        uses: pnpm/action-setup@v4
+        with:
+          version: 9
+          run_install: false
+
+      - name: Setup Node.js Version 20 (LTS)
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'pnpm'
+
+      - name: Install Dependencies and Sync Content
+        run: |
+          echo "Installing Dependencies"
+          pnpm install
+          echo "Syncing Content to Repo"
+          npm run sync:content-to-repo -- --roadmap-slug=${{ inputs.roadmap_slug }} --secret=${{ secrets.GH_SYNC_SECRET }}
+
+      - name: Check for changes
+        id: verify-changed-files
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "changed=false" >> $GITHUB_OUTPUT
+          fi
+      
+      - name: Create PR
+        if: steps.verify-changed-files.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          delete-branch: false
+          branch: "chore/sync-content-to-repo-${{ inputs.roadmap_slug }}"
+          base: "master"
+          labels: |
+            dependencies
+            automated pr
+          reviewers: arikchakma
+          commit-message: "chore: sync content to repo"
+          title: "Sync Content to Repo - Automated"
+          body: |
+            ## Sync Content to Repo
+            
+            > [!IMPORTANT]
+            > This PR Syncs the Content to the Repo for the Roadmap: ${{ inputs.roadmap_slug }}
+            >
+            > Commit: ${{ github.sha }}
+            > Workflow Path: ${{ github.workflow_ref }}
+
+            **Please Review the Changes and Merge the PR if everything is fine.** 

.github/workflows/sync-repo-to-database.yml

@@ -0,0 +1,66 @@
+name: Sync on Roadmap Changes
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'src/data/roadmaps/**'
+
+jobs:
+  sync-on-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2  # Fetch previous commit to compare changes
+
+      - name: Setup pnpm@v9
+        uses: pnpm/action-setup@v4
+        with:
+          version: 9
+          run_install: false
+
+      - name: Setup Node.js Version 20 (LTS)
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'pnpm'
+
+      - name: Get changed files
+        id: changed-files
+        run: |
+          echo "Getting changed files in /src/data/roadmaps/"
+          
+          # Get changed files between HEAD and previous commit
+          CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD -- src/data/roadmaps/)
+          
+          if [ -z "$CHANGED_FILES" ]; then
+            echo "No changes found in roadmaps directory"
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+          
+          echo "Changed files:"
+          echo "$CHANGED_FILES"
+          
+          # Convert to space-separated list for the script
+          CHANGED_FILES_LIST=$(echo "$CHANGED_FILES" | tr '\n' ' ')
+          
+          echo "has_changes=true" >> $GITHUB_OUTPUT
+          echo "changed_files=$CHANGED_FILES_LIST" >> $GITHUB_OUTPUT
+
+      - name: Install Dependencies
+        if: steps.changed-files.outputs.has_changes == 'true'
+        run: |
+          echo "Installing Dependencies"
+          pnpm install
+
+      - name: Run sync script with changed files
+        if: steps.changed-files.outputs.has_changes == 'true'
+        run: |
+          echo "Running sync script for changed roadmap files"
+          echo "Changed files: ${{ steps.changed-files.outputs.changed_files }}"
+          
+          # Run your script with the changed file paths
+          npm run sync:repo-to-database -- --files="${{ steps.changed-files.outputs.changed_files }}" --secret=${{ secrets.TOPIC_CONTENT_SYNC_SECRET }}

package.json

@@ -29,6 +29,8 @@
     "compress:images": "tsx ./scripts/compress-images.ts",
     "generate:roadmap-content-json": "tsx ./scripts/editor-roadmap-content-json.ts",
     "migrate:editor-roadmaps": "tsx ./scripts/migrate-editor-roadmap.ts",
+    "sync:content-to-repo": "tsx ./scripts/sync-content-to-repo.ts",
+    "sync:repo-to-database": "tsx ./scripts/sync-repo-to-database.ts",
     "test:e2e": "playwright test"
   },
   "dependencies": {

scripts/sync-content-to-repo.ts

@@ -0,0 +1,110 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { slugify } from '../src/lib/slugger';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const args = process.argv.slice(2);
+const roadmapSlug = args?.[0]?.replace('--roadmap-slug=', '');
+const secret = args?.[1]?.replace('--secret=', '');
+if (!secret) {
+  throw new Error('Secret is required');
+}
+
+if (!roadmapSlug || roadmapSlug === '__default__') {
+  throw new Error('Roadmap slug is required');
+}
+
+console.log(`🚀 Starting ${roadmapSlug}`);
+export const allowedOfficialRoadmapTopicResourceType = [
+  'roadmap',
+  'official',
+  'opensource',
+  'article',
+  'course',
+  'podcast',
+  'video',
+  'book',
+  'feed',
+] as const;
+export type AllowedOfficialRoadmapTopicResourceType =
+  (typeof allowedOfficialRoadmapTopicResourceType)[number];
+
+export type OfficialRoadmapTopicResource = {
+  _id?: string;
+  type: AllowedOfficialRoadmapTopicResourceType;
+  title: string;
+  url: string;
+};
+
+export interface OfficialRoadmapTopicContentDocument {
+  _id?: string;
+  roadmapSlug: string;
+  nodeId: string;
+  title: string;
+  description: string;
+  resources: OfficialRoadmapTopicResource[];
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+export async function roadmapTopics(
+  roadmapId: string,
+  secret: string,
+): Promise<OfficialRoadmapTopicContentDocument[]> {
+  const path = `https://roadmap.sh/api/v1-list-official-roadmap-topics/${roadmapId}?secret=${secret}`;
+  const response = await fetch(path);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch roadmap topics: ${response.statusText}`);
+  }
+
+  const data = await response.json();
+  if (data.error) {
+    throw new Error(`Failed to fetch roadmap topics: ${data.error}`);
+  }
+
+  return data;
+}
+
+// Directory containing the roadmaps
+const ROADMAP_CONTENT_DIR = path.join(
+  __dirname,
+  '../src/data/roadmaps',
+  roadmapSlug,
+);
+
+const allTopics = await roadmapTopics(roadmapSlug, secret);
+for (const topic of allTopics) {
+  const { title, nodeId } = topic;
+
+  const topicSlug = `${slugify(title)}@${nodeId}.md`;
+
+  const topicPath = path.join(ROADMAP_CONTENT_DIR, topicSlug);
+  const topicDir = path.dirname(topicPath);
+  const topicDirExists = await fs
+    .stat(topicDir)
+    .then(() => true)
+    .catch(() => false);
+  if (!topicDirExists) {
+    await fs.mkdir(topicDir, { recursive: true });
+  }
+
+  const topicContent = prepareTopicContent(topic);
+  await fs.writeFile(topicPath, topicContent);
+  console.log(`✅ Synced ${topicSlug}`);
+}
+
+function prepareTopicContent(topic: OfficialRoadmapTopicContentDocument) {
+  const { description, resources = [] } = topic;
+
+  const content = `${description}
+
+Visit the following resources to learn more:
+
+${resources.map((resource) => `- [@${resource.type}@${resource.title}](${resource.url})`).join('\n')}
+  `.trim();
+
+  return content;
+}

scripts/sync-repo-to-database.ts

@@ -0,0 +1,218 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { slugify } from '../src/lib/slugger';
+import type { OfficialRoadmapDocument } from '../src/queries/official-roadmap';
+import { parse } from 'node-html-parser';
+import { markdownToHtml } from '../src/lib/markdown';
+import { htmlToMarkdown } from '../src/lib/html';
+import {
+  allowedOfficialRoadmapTopicResourceType,
+  type AllowedOfficialRoadmapTopicResourceType,
+  type OfficialRoadmapTopicContentDocument,
+  type OfficialRoadmapTopicResource,
+} from './sync-content-to-repo';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const args = process.argv.slice(2);
+const allFiles = args?.[0]?.replace('--files=', '');
+const secret = args?.[1]?.replace('--secret=', '');
+if (!secret) {
+  throw new Error('Secret is required');
+}
+
+let roadmapJsonCache: Map<string, OfficialRoadmapDocument> = new Map();
+export async function fetchRoadmapJson(
+  roadmapId: string,
+): Promise<OfficialRoadmapDocument> {
+  if (roadmapJsonCache.has(roadmapId)) {
+    return roadmapJsonCache.get(roadmapId)!;
+  }
+
+  const response = await fetch(
+    `https://roadmap.sh/api/v1-official-roadmap/${roadmapId}`,
+  );
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch roadmap json: ${response.statusText}`);
+  }
+
+  const data = await response.json();
+  if (data.error) {
+    throw new Error(`Failed to fetch roadmap json: ${data.error}`);
+  }
+
+  roadmapJsonCache.set(roadmapId, data);
+  return data;
+}
+
+export async function syncContentToDatabase(
+  topics: Omit<
+    OfficialRoadmapTopicContentDocument,
+    'createdAt' | 'updatedAt' | '_id'
+  >[],
+) {
+  const response = await fetch(
+    `https://roadmap.sh/api/v1-sync-official-roadmap-topics`,
+    {
+      method: 'POST',
+      body: JSON.stringify({
+        topics,
+        secret,
+      }),
+    },
+  );
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to sync content to database: ${response.statusText}`,
+    );
+  }
+
+  return response.json();
+}
+
+const files = allFiles.split(' ');
+console.log(`🚀 Starting ${files.length} files`);
+
+const ROADMAP_CONTENT_DIR = path.join(__dirname, '../src/data/roadmaps');
+
+try {
+  const topics: Omit<
+    OfficialRoadmapTopicContentDocument,
+    'createdAt' | 'updatedAt' | '_id'
+  >[] = [];
+
+  for (const file of files) {
+    const isContentFile = file.endsWith('.md') && file.includes('content/');
+    if (!isContentFile) {
+      console.log(`🚨 Skipping ${file} because it is not a content file`);
+      continue;
+    }
+
+    const pathParts = file.replace('src/data/roadmaps/', '').split('/');
+    const roadmapSlug = pathParts?.[0];
+    if (!roadmapSlug) {
+      console.error(`🚨 Roadmap slug is required: ${file}`);
+      continue;
+    }
+
+    const nodeSlug = pathParts?.[2]?.replace('.md', '');
+    if (!nodeSlug) {
+      console.error(`🚨 Node id is required: ${file}`);
+      continue;
+    }
+
+    const nodeId = nodeSlug.split('@')?.[1];
+    if (!nodeId) {
+      console.error(`🚨 Node id is required: ${file}`);
+      continue;
+    }
+
+    const roadmap = await fetchRoadmapJson(roadmapSlug);
+    const node = roadmap.nodes.find((node) => node.id === nodeId);
+    if (!node) {
+      console.error(`🚨 Node not found: ${file}`);
+      continue;
+    }
+
+    const filePath = path.join(
+      ROADMAP_CONTENT_DIR,
+      roadmapSlug,
+      'content',
+      `${nodeSlug}.md`,
+    );
+
+    const content = await fs.readFile(filePath, 'utf8');
+    const html = markdownToHtml(content, false);
+    const rootHtml = parse(html);
+
+    let ulWithLinks: HTMLElement | undefined;
+    rootHtml.querySelectorAll('ul').forEach((ul) => {
+      const listWithJustLinks = Array.from(ul.querySelectorAll('li')).filter(
+        (li) => {
+          const link = li.querySelector('a');
+          return link && link.textContent?.trim() === li.textContent?.trim();
+        },
+      );
+
+      if (listWithJustLinks.length > 0) {
+        // @ts-expect-error - TODO: fix this
+        ulWithLinks = ul;
+      }
+    });
+
+    const listLinks: Omit<OfficialRoadmapTopicResource, '_id'>[] =
+      ulWithLinks !== undefined
+        ? Array.from(ulWithLinks.querySelectorAll('li > a'))
+            .map((link) => {
+              const typePattern = /@([a-z.]+)@/;
+              let linkText = link.textContent || '';
+              const linkHref = link.getAttribute('href') || '';
+              let linkType = linkText.match(typePattern)?.[1] || 'article';
+              linkType = allowedOfficialRoadmapTopicResourceType.includes(
+                linkType as any,
+              )
+                ? linkType
+                : 'article';
+
+              linkText = linkText.replace(typePattern, '');
+
+              return {
+                title: linkText,
+                url: linkHref,
+                type: linkType as AllowedOfficialRoadmapTopicResourceType,
+              };
+            })
+            .sort((a, b) => {
+              const order = [
+                'official',
+                'opensource',
+                'article',
+                'video',
+                'feed',
+              ];
+              return order.indexOf(a.type) - order.indexOf(b.type);
+            })
+        : [];
+
+    const title = rootHtml.querySelector('h1');
+    ulWithLinks?.remove();
+    title?.remove();
+
+    const allParagraphs = rootHtml.querySelectorAll('p');
+    if (listLinks.length > 0 && allParagraphs.length > 0) {
+      // to remove the view more see more from the description
+      const lastParagraph = allParagraphs[allParagraphs.length - 1];
+      lastParagraph?.remove();
+    }
+
+    const htmlStringWithoutLinks = rootHtml.toString();
+    const description = htmlToMarkdown(htmlStringWithoutLinks);
+
+    const updatedDescription = `# ${title?.textContent}
+
+${description}`.trim();
+
+    const label = node?.data?.label as string;
+    if (!label) {
+      console.error(`🚨 Label is required: ${file}`);
+      continue;
+    }
+
+    topics.push({
+      roadmapSlug,
+      nodeId,
+      title: label,
+      description: updatedDescription,
+      resources: listLinks,
+    });
+  }
+
+  await syncContentToDatabase(topics);
+} catch (error) {
+  console.error(error);
+  process.exit(1);
+}

src/queries/official-roadmap.ts

@@ -1,13 +1,14 @@
 import { queryOptions } from '@tanstack/react-query';
 import { httpGet } from '../lib/query-http';
+import type { Node, Edge } from '@roadmapsh/editor';
 
 export interface OfficialRoadmapDocument {
   _id: string;
   title: string;
   description?: string;
   slug: string;
-  nodes: any[];
-  edges: any[];
+  nodes: Node[];
+  edges: Edge[];
 
   createdAt: Date;
   updatedAt: Date;