Add script to import data from JSON files into the database
Creates a new TypeScript script `scripts/import-data.ts` that reads data from a JSON file and imports it into the database, supporting various tables including sessions, users, media outlets, articles, prediction markets, auctions, bids, and more. Replit-Commit-Author: Agent Replit-Commit-Session-Id: aabe2db1-f078-4501-aab5-be145ebc6b9a Replit-Commit-Checkpoint-Type: intermediate_checkpoint Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/3df548ff-50ae-432f-9be4-25d34eccc983/aabe2db1-f078-4501-aab5-be145ebc6b9a/TqVS1Ec
This commit is contained in:
kimjaehyeon0101
175
scripts/import-data.ts
Normal file
175
scripts/import-data.ts
Normal file
@ -0,0 +1,175 @@
|
|||||||
|
import { drizzle } from "drizzle-orm/neon-serverless";
|
||||||
|
import { Pool } from "@neondatabase/serverless";
|
||||||
|
import {
|
||||||
|
sessions,
|
||||||
|
users,
|
||||||
|
mediaOutlets,
|
||||||
|
articles,
|
||||||
|
predictionMarkets,
|
||||||
|
auctions,
|
||||||
|
bids,
|
||||||
|
mediaOutletRequests,
|
||||||
|
comments,
|
||||||
|
predictionBets,
|
||||||
|
} from "../shared/schema";
|
||||||
|
import { readFileSync } from "fs";
|
||||||
|
import { join } from "path";
|
||||||
|
|
||||||
|
interface ExportData {
|
||||||
|
exportDate: string;
|
||||||
|
version: string;
|
||||||
|
tables: {
|
||||||
|
sessions: any[];
|
||||||
|
users: any[];
|
||||||
|
mediaOutlets: any[];
|
||||||
|
articles: any[];
|
||||||
|
predictionMarkets: any[];
|
||||||
|
auctions: any[];
|
||||||
|
bids: any[];
|
||||||
|
mediaOutletRequests: any[];
|
||||||
|
comments: any[];
|
||||||
|
predictionBets: any[];
|
||||||
|
};
|
||||||
|
metadata: {
|
||||||
|
totalRecords: number;
|
||||||
|
tableStats: Record<string, number>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function importData() {
|
||||||
|
console.log("🚀 Starting data import...\n");
|
||||||
|
|
||||||
|
// Get command line arguments
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const filename = args[0];
|
||||||
|
const targetDatabaseUrl = args[1];
|
||||||
|
|
||||||
|
if (!filename) {
|
||||||
|
console.error("❌ Error: No filename provided");
|
||||||
|
console.log("\nUsage: npm run db:import <filename> [database_url]");
|
||||||
|
console.log("\nExample:");
|
||||||
|
console.log(" npm run db:import data-export-2025-10-13.json");
|
||||||
|
console.log(" npm run db:import data-export-2025-10-13.json postgresql://user:pass@host/db");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read export file
|
||||||
|
const filepath = join(process.cwd(), filename);
|
||||||
|
console.log(`📂 Reading data from ${filename}...`);
|
||||||
|
|
||||||
|
let exportData: ExportData;
|
||||||
|
try {
|
||||||
|
const fileContent = readFileSync(filepath, 'utf-8');
|
||||||
|
exportData = JSON.parse(fileContent);
|
||||||
|
console.log(` ✅ File loaded successfully`);
|
||||||
|
console.log(` 📅 Export date: ${exportData.exportDate}`);
|
||||||
|
console.log(` 📊 Total records: ${exportData.metadata.totalRecords}`);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`❌ Error reading file:`, error);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Setup database connection
|
||||||
|
const databaseUrl = targetDatabaseUrl || process.env.DATABASE_URL;
|
||||||
|
if (!databaseUrl) {
|
||||||
|
console.error("❌ Error: No database URL provided");
|
||||||
|
console.log("\nPlease provide a DATABASE_URL environment variable or pass it as an argument");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n🔗 Connecting to database...`);
|
||||||
|
const pool = new Pool({ connectionString: databaseUrl });
|
||||||
|
const targetDb = drizzle(pool);
|
||||||
|
console.log(` ✅ Connected successfully`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Import data in dependency order
|
||||||
|
console.log("\n📥 Importing data...\n");
|
||||||
|
|
||||||
|
// 1. Sessions
|
||||||
|
if (exportData.tables.sessions.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.sessions.length} sessions...`);
|
||||||
|
await targetDb.insert(sessions).values(exportData.tables.sessions).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Sessions imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Users
|
||||||
|
if (exportData.tables.users.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.users.length} users...`);
|
||||||
|
await targetDb.insert(users).values(exportData.tables.users).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Users imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Media Outlets
|
||||||
|
if (exportData.tables.mediaOutlets.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.mediaOutlets.length} media outlets...`);
|
||||||
|
await targetDb.insert(mediaOutlets).values(exportData.tables.mediaOutlets).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Media outlets imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Articles
|
||||||
|
if (exportData.tables.articles.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.articles.length} articles...`);
|
||||||
|
await targetDb.insert(articles).values(exportData.tables.articles).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Articles imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Prediction Markets
|
||||||
|
if (exportData.tables.predictionMarkets.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.predictionMarkets.length} prediction markets...`);
|
||||||
|
await targetDb.insert(predictionMarkets).values(exportData.tables.predictionMarkets).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Prediction markets imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. Auctions
|
||||||
|
if (exportData.tables.auctions.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.auctions.length} auctions...`);
|
||||||
|
await targetDb.insert(auctions).values(exportData.tables.auctions).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Auctions imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 7. Bids
|
||||||
|
if (exportData.tables.bids.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.bids.length} bids...`);
|
||||||
|
await targetDb.insert(bids).values(exportData.tables.bids).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Bids imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 8. Media Outlet Requests
|
||||||
|
if (exportData.tables.mediaOutletRequests.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.mediaOutletRequests.length} media outlet requests...`);
|
||||||
|
await targetDb.insert(mediaOutletRequests).values(exportData.tables.mediaOutletRequests).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Media outlet requests imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 9. Comments
|
||||||
|
if (exportData.tables.comments.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.comments.length} comments...`);
|
||||||
|
await targetDb.insert(comments).values(exportData.tables.comments).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Comments imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 10. Prediction Bets
|
||||||
|
if (exportData.tables.predictionBets.length > 0) {
|
||||||
|
console.log(`📊 Importing ${exportData.tables.predictionBets.length} prediction bets...`);
|
||||||
|
await targetDb.insert(predictionBets).values(exportData.tables.predictionBets).onConflictDoNothing();
|
||||||
|
console.log(` ✅ Prediction bets imported`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n✅ Import completed successfully!`);
|
||||||
|
console.log(`\n📊 Import summary:`);
|
||||||
|
Object.entries(exportData.metadata.tableStats).forEach(([table, count]) => {
|
||||||
|
console.log(` - ${table}: ${count} records`);
|
||||||
|
});
|
||||||
|
|
||||||
|
await pool.end();
|
||||||
|
process.exit(0);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("\n❌ Error during import:", error);
|
||||||
|
await pool.end();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run import
|
||||||
|
importData();
|
||||||
Reference in New Issue
Block a user