333 lines
9.0 KiB
JavaScript
333 lines
9.0 KiB
JavaScript
/**
|
||
* 批量处理器
|
||
* 支持5000+ SKU的批量图片生成
|
||
*/
|
||
|
||
const axios = require('axios');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
/**
|
||
* 批量处理配置
|
||
*/
|
||
const DEFAULT_CONFIG = {
|
||
concurrency: 1, // 并发数（建议保持1避免API限流）
|
||
retryCount: 3, // 失败重试次数
|
||
retryDelay: 5000, // 重试延迟（毫秒）
|
||
imageDelay: 1500, // 图片间延迟（毫秒）
|
||
skuDelay: 3000, // SKU间延迟（毫秒）
|
||
saveProgress: true, // 是否保存进度
|
||
progressFile: 'batch-progress.json'
|
||
};
|
||
|
||
/**
|
||
* 批量处理器类
|
||
*/
|
||
class BatchProcessor {
|
||
constructor(serverUrl, config = {}) {
|
||
this.serverUrl = serverUrl || 'http://localhost:3000';
|
||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||
this.progress = {
|
||
total: 0,
|
||
completed: 0,
|
||
failed: 0,
|
||
results: []
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 处理单个SKU
|
||
* @param {object} sku - SKU数据
|
||
* @returns {Promise<object>} 处理结果
|
||
*/
|
||
async processSKU(sku) {
|
||
const startTime = Date.now();
|
||
|
||
try {
|
||
console.log(`Processing SKU: ${sku.sku_id}`);
|
||
|
||
const response = await axios.post(`${this.serverUrl}/api/generate-sku`, {
|
||
sku
|
||
}, {
|
||
timeout: 600000 // 10分钟超时（12张图）
|
||
});
|
||
|
||
const duration = (Date.now() - startTime) / 1000;
|
||
console.log(`SKU ${sku.sku_id} completed in ${duration.toFixed(1)}s`);
|
||
|
||
return {
|
||
sku_id: sku.sku_id,
|
||
status: 'success',
|
||
duration,
|
||
...response.data
|
||
};
|
||
|
||
} catch (error) {
|
||
const duration = (Date.now() - startTime) / 1000;
|
||
console.error(`SKU ${sku.sku_id} failed after ${duration.toFixed(1)}s:`, error.message);
|
||
|
||
return {
|
||
sku_id: sku.sku_id,
|
||
status: 'failed',
|
||
duration,
|
||
error: error.message
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 带重试的处理单个SKU
|
||
* @param {object} sku - SKU数据
|
||
* @returns {Promise<object>} 处理结果
|
||
*/
|
||
async processSKUWithRetry(sku) {
|
||
let lastError;
|
||
|
||
for (let attempt = 1; attempt <= this.config.retryCount; attempt++) {
|
||
try {
|
||
const result = await this.processSKU(sku);
|
||
|
||
if (result.status === 'success') {
|
||
return result;
|
||
}
|
||
|
||
lastError = result.error;
|
||
|
||
if (attempt < this.config.retryCount) {
|
||
console.log(`Retrying SKU ${sku.sku_id} (attempt ${attempt + 1}/${this.config.retryCount})...`);
|
||
await this.delay(this.config.retryDelay);
|
||
}
|
||
} catch (error) {
|
||
lastError = error.message;
|
||
|
||
if (attempt < this.config.retryCount) {
|
||
console.log(`Retrying SKU ${sku.sku_id} (attempt ${attempt + 1}/${this.config.retryCount})...`);
|
||
await this.delay(this.config.retryDelay);
|
||
}
|
||
}
|
||
}
|
||
|
||
return {
|
||
sku_id: sku.sku_id,
|
||
status: 'failed',
|
||
error: lastError,
|
||
attempts: this.config.retryCount
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 批量处理SKU列表
|
||
* @param {array} skus - SKU数据数组
|
||
* @param {function} onProgress - 进度回调
|
||
* @returns {Promise<object>} 批量处理结果
|
||
*/
|
||
async processBatch(skus, onProgress = null) {
|
||
this.progress = {
|
||
total: skus.length,
|
||
completed: 0,
|
||
failed: 0,
|
||
startTime: Date.now(),
|
||
results: []
|
||
};
|
||
|
||
console.log(`\n${'='.repeat(60)}`);
|
||
console.log(`Starting batch processing: ${skus.length} SKUs`);
|
||
console.log(`${'='.repeat(60)}\n`);
|
||
|
||
// 加载之前的进度（如果有）
|
||
const existingProgress = this.loadProgress();
|
||
const processedIds = new Set(existingProgress.map(r => r.sku_id));
|
||
|
||
// 过滤掉已处理的SKU
|
||
const pendingSkus = skus.filter(sku => !processedIds.has(sku.sku_id));
|
||
this.progress.results = existingProgress;
|
||
this.progress.completed = existingProgress.filter(r => r.status === 'success').length;
|
||
this.progress.failed = existingProgress.filter(r => r.status === 'failed').length;
|
||
|
||
if (existingProgress.length > 0) {
|
||
console.log(`Resuming from previous progress: ${existingProgress.length} already processed`);
|
||
}
|
||
|
||
// 串行处理（避免API限流）
|
||
for (let i = 0; i < pendingSkus.length; i++) {
|
||
const sku = pendingSkus[i];
|
||
const overallIndex = existingProgress.length + i + 1;
|
||
|
||
console.log(`\n[${overallIndex}/${skus.length}] Processing ${sku.sku_id}...`);
|
||
|
||
const result = await this.processSKUWithRetry(sku);
|
||
this.progress.results.push(result);
|
||
|
||
if (result.status === 'success') {
|
||
this.progress.completed++;
|
||
} else {
|
||
this.progress.failed++;
|
||
}
|
||
|
||
// 保存进度
|
||
if (this.config.saveProgress) {
|
||
this.saveProgress();
|
||
}
|
||
|
||
// 进度回调
|
||
if (onProgress) {
|
||
onProgress({
|
||
current: overallIndex,
|
||
total: skus.length,
|
||
completed: this.progress.completed,
|
||
failed: this.progress.failed,
|
||
lastResult: result
|
||
});
|
||
}
|
||
|
||
// SKU间延迟
|
||
if (i < pendingSkus.length - 1) {
|
||
await this.delay(this.config.skuDelay);
|
||
}
|
||
}
|
||
|
||
this.progress.endTime = Date.now();
|
||
this.progress.totalDuration = (this.progress.endTime - this.progress.startTime) / 1000;
|
||
|
||
console.log(`\n${'='.repeat(60)}`);
|
||
console.log(`Batch processing complete!`);
|
||
console.log(`Total: ${skus.length}, Success: ${this.progress.completed}, Failed: ${this.progress.failed}`);
|
||
console.log(`Duration: ${this.progress.totalDuration.toFixed(1)}s`);
|
||
console.log(`${'='.repeat(60)}\n`);
|
||
|
||
return {
|
||
total: skus.length,
|
||
completed: this.progress.completed,
|
||
failed: this.progress.failed,
|
||
duration: this.progress.totalDuration,
|
||
results: this.progress.results
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 延迟函数
|
||
* @param {number} ms - 毫秒数
|
||
*/
|
||
delay(ms) {
|
||
return new Promise(resolve => setTimeout(resolve, ms));
|
||
}
|
||
|
||
/**
|
||
* 保存进度到文件
|
||
*/
|
||
saveProgress() {
|
||
try {
|
||
const progressPath = path.join(process.cwd(), this.config.progressFile);
|
||
fs.writeFileSync(progressPath, JSON.stringify(this.progress.results, null, 2));
|
||
} catch (error) {
|
||
console.error('Failed to save progress:', error.message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 加载之前的进度
|
||
* @returns {array} 之前的处理结果
|
||
*/
|
||
loadProgress() {
|
||
try {
|
||
const progressPath = path.join(process.cwd(), this.config.progressFile);
|
||
if (fs.existsSync(progressPath)) {
|
||
const content = fs.readFileSync(progressPath, 'utf-8');
|
||
return JSON.parse(content);
|
||
}
|
||
} catch (error) {
|
||
console.error('Failed to load progress:', error.message);
|
||
}
|
||
return [];
|
||
}
|
||
|
||
/**
|
||
* 清除进度文件
|
||
*/
|
||
clearProgress() {
|
||
try {
|
||
const progressPath = path.join(process.cwd(), this.config.progressFile);
|
||
if (fs.existsSync(progressPath)) {
|
||
fs.unlinkSync(progressPath);
|
||
console.log('Progress cleared');
|
||
}
|
||
} catch (error) {
|
||
console.error('Failed to clear progress:', error.message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 生成批处理报告
|
||
* @returns {object} 报告数据
|
||
*/
|
||
generateReport() {
|
||
const successResults = this.progress.results.filter(r => r.status === 'success');
|
||
const failedResults = this.progress.results.filter(r => r.status === 'failed');
|
||
|
||
const totalImages = successResults.reduce((sum, r) => {
|
||
return sum + (r.summary?.success || 0);
|
||
}, 0);
|
||
|
||
return {
|
||
summary: {
|
||
total_skus: this.progress.total,
|
||
successful_skus: this.progress.completed,
|
||
failed_skus: this.progress.failed,
|
||
success_rate: ((this.progress.completed / this.progress.total) * 100).toFixed(1) + '%',
|
||
total_images_generated: totalImages,
|
||
total_duration: this.progress.totalDuration?.toFixed(1) + 's',
|
||
average_time_per_sku: (this.progress.totalDuration / this.progress.total).toFixed(1) + 's'
|
||
},
|
||
failed_skus: failedResults.map(r => ({
|
||
sku_id: r.sku_id,
|
||
error: r.error
|
||
})),
|
||
successful_skus: successResults.map(r => ({
|
||
sku_id: r.sku_id,
|
||
images: r.summary
|
||
}))
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 从文件加载SKU列表
|
||
* @param {string} filePath - 文件路径（JSON或CSV）
|
||
* @returns {array} SKU数组
|
||
*/
|
||
function loadSKUsFromFile(filePath) {
|
||
const content = fs.readFileSync(filePath, 'utf-8');
|
||
const ext = path.extname(filePath).toLowerCase();
|
||
|
||
if (ext === '.json') {
|
||
const data = JSON.parse(content);
|
||
return Array.isArray(data) ? data : [data];
|
||
}
|
||
|
||
// 简单CSV解析（假设第一行是表头）
|
||
if (ext === '.csv') {
|
||
const lines = content.split('\n').filter(line => line.trim());
|
||
const headers = lines[0].split(',').map(h => h.trim());
|
||
|
||
return lines.slice(1).map(line => {
|
||
const values = line.split(',').map(v => v.trim());
|
||
const obj = {};
|
||
headers.forEach((h, i) => {
|
||
obj[h] = values[i];
|
||
});
|
||
return obj;
|
||
});
|
||
}
|
||
|
||
throw new Error('Unsupported file format. Use .json or .csv');
|
||
}
|
||
|
||
module.exports = {
|
||
BatchProcessor,
|
||
loadSKUsFromFile,
|
||
DEFAULT_CONFIG
|
||
};
|
||
|
||
|
||
|