Add sync roadmap to database

package.json

@@ -31,6 +31,7 @@
     "migrate:editor-roadmaps": "tsx ./scripts/migrate-editor-roadmap.ts",
     "sync:content-to-repo": "tsx ./scripts/sync-content-to-repo.ts",
     "sync:repo-to-database": "tsx ./scripts/sync-repo-to-database.ts",
+    "sync:roadmap": "tsx ./scripts/sync-roadmap-to-database.ts",
     "migrate:content-repo-to-database": "tsx ./scripts/migrate-content-repo-to-database.ts",
     "official:roadmap-assets": "tsx ./scripts/official-roadmap-assets.ts",
     "test:e2e": "playwright test"

scripts/sync-repo-to-database.ts

@@ -36,6 +36,11 @@ export async function fetchRoadmapJson(
 
   const response = await fetch(
     `https://roadmap.sh/api/v1-official-roadmap/${roadmapId}`,
+    {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (compatible; roadmap-sync/1.0)',
+      },
+    },
   );
 
   if (!response.ok) {
@@ -64,6 +69,7 @@ export async function syncContentToDatabase(
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
+        'User-Agent': 'Mozilla/5.0 (compatible; roadmap-sync/1.0)',
       },
       body: JSON.stringify({
         topics,
@@ -72,14 +78,21 @@ export async function syncContentToDatabase(
     },
   );
 
+  const responseText = await response.text();
+
   if (!response.ok) {
-    const error = await response.json();
     throw new Error(
-      `Failed to sync content to database: ${response.statusText} ${JSON.stringify(error, null, 2)}`,
+      `Failed to sync content to database: ${response.status} ${response.statusText}\n${responseText}`,
     );
   }
 
-  return response.json();
+  try {
+    return JSON.parse(responseText);
+  } catch {
+    throw new Error(
+      `Failed to parse response as JSON: ${responseText.substring(0, 500)}`,
+    );
+  }
 }
 
 const files =
@@ -231,8 +244,15 @@ try {
     });
   }
 
+  console.log(`📤 Syncing ${topics.length} topics to database...`);
   await syncContentToDatabase(topics);
+  console.log(`✅ Successfully synced ${topics.length} topics`);
 } catch (error) {
+  console.error('❌ Sync failed with error:');
   console.error(error);
+  if (error instanceof Error) {
+    console.error('\nError message:', error.message);
+    console.error('\nStack trace:', error.stack);
+  }
   process.exit(1);
 }

scripts/sync-roadmap-to-database.ts

@@ -0,0 +1,82 @@
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const args = process.argv.slice(2);
+const roadmapSlug = args
+  .find((arg) => arg.startsWith('--roadmap='))
+  ?.replace('--roadmap=', '');
+const secret = args
+  .find((arg) => arg.startsWith('--secret='))
+  ?.replace('--secret=', '');
+
+if (!roadmapSlug) {
+  console.error('❌ Roadmap slug is required. Use --roadmap=<slug>');
+  console.error('   Example: npm run sync:roadmap -- --roadmap=frontend --secret=<secret>');
+  process.exit(1);
+}
+
+if (!secret) {
+  console.error('❌ Secret is required. Use --secret=<secret>');
+  console.error('   Example: npm run sync:roadmap -- --roadmap=frontend --secret=<secret>');
+  process.exit(1);
+}
+
+const roadmapDir = path.join(__dirname, '../src/data/roadmaps', roadmapSlug);
+
+if (!fs.existsSync(roadmapDir)) {
+  console.error(`❌ Roadmap directory not found: ${roadmapDir}`);
+  process.exit(1);
+}
+
+console.log(`🔍 Finding all content files in: ${roadmapDir}`);
+
+function getAllFiles(dir: string): string[] {
+  const files: string[] = [];
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...getAllFiles(fullPath));
+    } else {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+const allFiles = getAllFiles(roadmapDir);
+const relativeFiles = allFiles.map((file) =>
+  file.replace(path.join(__dirname, '../'), ''),
+);
+
+console.log(`📁 Found ${relativeFiles.length} files`);
+
+if (relativeFiles.length === 0) {
+  console.log('⚠️ No files found to sync');
+  process.exit(0);
+}
+
+const filesArg = relativeFiles.join(',');
+
+console.log(`🚀 Syncing roadmap "${roadmapSlug}" to database...`);
+
+try {
+  execSync(
+    `npx tsx ./scripts/sync-repo-to-database.ts --files="${filesArg}" --secret=${secret}`,
+    {
+      cwd: path.join(__dirname, '..'),
+      stdio: 'inherit',
+    },
+  );
+  console.log(`✅ Successfully synced roadmap "${roadmapSlug}" to database`);
+} catch (error) {
+  console.error(`❌ Failed to sync roadmap "${roadmapSlug}" to database`);
+  process.exit(1);
+}