sapiens-mobile/find_duplicates.cjs

const fs = require('fs');

// Read the articles data
const articles = JSON.parse(fs.readFileSync('articles_temp.json', 'utf8'));

// Count thumbnail usage
const thumbnailCounts = {};
const thumbnailToArticles = {};

articles.forEach(article => {
    const thumbnail = article.thumbnail || '';
    if (thumbnail) {
        thumbnailCounts[thumbnail] = (thumbnailCounts[thumbnail] || 0) + 1;
        if (!thumbnailToArticles[thumbnail]) {
            thumbnailToArticles[thumbnail] = [];
        }
        thumbnailToArticles[thumbnail].push({
            id: article.id,
            title: article.title.length > 60 ? article.title.substring(0, 60) + '...' : article.title,
            outletId: article.outletId
        });
    }
});

// Find duplicates
console.log("=== DUPLICATE THUMBNAILS ===");
let duplicatesFound = false;
Object.entries(thumbnailCounts)
    .sort((a, b) => b[1] - a[1])
    .forEach(([thumbnail, count]) => {
        if (count > 1) {
            duplicatesFound = true;
            console.log(`\n${thumbnail} (used ${count} times):`);
            thumbnailToArticles[thumbnail].forEach(article => {
                console.log(`  - ${article.outletId}: ${article.title} (ID: ${article.id})`);
            });
        }
    });

if (!duplicatesFound) {
    console.log("No duplicate thumbnails found!");
}

// Show some stats
console.log(`\n=== STATISTICS ===`);
console.log(`Total articles: ${articles.length}`);
console.log(`Unique thumbnails: ${Object.keys(thumbnailCounts).length}`);
console.log(`Articles with duplicates: ${Object.values(thumbnailCounts).reduce((sum, count) => sum + (count > 1 ? count : 0), 0)}`);