Upload latest code and optimized prompts (v6)
This commit is contained in:
tony
332
lib/batch-processor.js
Normal file
332
lib/batch-processor.js
Normal file
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* 批量处理器
|
||||
* 支持5000+ SKU的批量图片生成
|
||||
*/
|
||||
|
||||
const axios = require('axios');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
/**
|
||||
* 批量处理配置
|
||||
*/
|
||||
const DEFAULT_CONFIG = {
|
||||
concurrency: 1, // 并发数（建议保持1避免API限流）
|
||||
retryCount: 3, // 失败重试次数
|
||||
retryDelay: 5000, // 重试延迟（毫秒）
|
||||
imageDelay: 1500, // 图片间延迟（毫秒）
|
||||
skuDelay: 3000, // SKU间延迟（毫秒）
|
||||
saveProgress: true, // 是否保存进度
|
||||
progressFile: 'batch-progress.json'
|
||||
};
|
||||
|
||||
/**
|
||||
* 批量处理器类
|
||||
*/
|
||||
class BatchProcessor {
|
||||
constructor(serverUrl, config = {}) {
|
||||
this.serverUrl = serverUrl || 'http://localhost:3000';
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
this.progress = {
|
||||
total: 0,
|
||||
completed: 0,
|
||||
failed: 0,
|
||||
results: []
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理单个SKU
|
||||
* @param {object} sku - SKU数据
|
||||
* @returns {Promise<object>} 处理结果
|
||||
*/
|
||||
async processSKU(sku) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
console.log(`Processing SKU: ${sku.sku_id}`);
|
||||
|
||||
const response = await axios.post(`${this.serverUrl}/api/generate-sku`, {
|
||||
sku
|
||||
}, {
|
||||
timeout: 600000 // 10分钟超时（12张图）
|
||||
});
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
console.log(`SKU ${sku.sku_id} completed in ${duration.toFixed(1)}s`);
|
||||
|
||||
return {
|
||||
sku_id: sku.sku_id,
|
||||
status: 'success',
|
||||
duration,
|
||||
...response.data
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
console.error(`SKU ${sku.sku_id} failed after ${duration.toFixed(1)}s:`, error.message);
|
||||
|
||||
return {
|
||||
sku_id: sku.sku_id,
|
||||
status: 'failed',
|
||||
duration,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 带重试的处理单个SKU
|
||||
* @param {object} sku - SKU数据
|
||||
* @returns {Promise<object>} 处理结果
|
||||
*/
|
||||
async processSKUWithRetry(sku) {
|
||||
let lastError;
|
||||
|
||||
for (let attempt = 1; attempt <= this.config.retryCount; attempt++) {
|
||||
try {
|
||||
const result = await this.processSKU(sku);
|
||||
|
||||
if (result.status === 'success') {
|
||||
return result;
|
||||
}
|
||||
|
||||
lastError = result.error;
|
||||
|
||||
if (attempt < this.config.retryCount) {
|
||||
console.log(`Retrying SKU ${sku.sku_id} (attempt ${attempt + 1}/${this.config.retryCount})...`);
|
||||
await this.delay(this.config.retryDelay);
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error.message;
|
||||
|
||||
if (attempt < this.config.retryCount) {
|
||||
console.log(`Retrying SKU ${sku.sku_id} (attempt ${attempt + 1}/${this.config.retryCount})...`);
|
||||
await this.delay(this.config.retryDelay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
sku_id: sku.sku_id,
|
||||
status: 'failed',
|
||||
error: lastError,
|
||||
attempts: this.config.retryCount
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量处理SKU列表
|
||||
* @param {array} skus - SKU数据数组
|
||||
* @param {function} onProgress - 进度回调
|
||||
* @returns {Promise<object>} 批量处理结果
|
||||
*/
|
||||
async processBatch(skus, onProgress = null) {
|
||||
this.progress = {
|
||||
total: skus.length,
|
||||
completed: 0,
|
||||
failed: 0,
|
||||
startTime: Date.now(),
|
||||
results: []
|
||||
};
|
||||
|
||||
console.log(`\n${'='.repeat(60)}`);
|
||||
console.log(`Starting batch processing: ${skus.length} SKUs`);
|
||||
console.log(`${'='.repeat(60)}\n`);
|
||||
|
||||
// 加载之前的进度（如果有）
|
||||
const existingProgress = this.loadProgress();
|
||||
const processedIds = new Set(existingProgress.map(r => r.sku_id));
|
||||
|
||||
// 过滤掉已处理的SKU
|
||||
const pendingSkus = skus.filter(sku => !processedIds.has(sku.sku_id));
|
||||
this.progress.results = existingProgress;
|
||||
this.progress.completed = existingProgress.filter(r => r.status === 'success').length;
|
||||
this.progress.failed = existingProgress.filter(r => r.status === 'failed').length;
|
||||
|
||||
if (existingProgress.length > 0) {
|
||||
console.log(`Resuming from previous progress: ${existingProgress.length} already processed`);
|
||||
}
|
||||
|
||||
// 串行处理（避免API限流）
|
||||
for (let i = 0; i < pendingSkus.length; i++) {
|
||||
const sku = pendingSkus[i];
|
||||
const overallIndex = existingProgress.length + i + 1;
|
||||
|
||||
console.log(`\n[${overallIndex}/${skus.length}] Processing ${sku.sku_id}...`);
|
||||
|
||||
const result = await this.processSKUWithRetry(sku);
|
||||
this.progress.results.push(result);
|
||||
|
||||
if (result.status === 'success') {
|
||||
this.progress.completed++;
|
||||
} else {
|
||||
this.progress.failed++;
|
||||
}
|
||||
|
||||
// 保存进度
|
||||
if (this.config.saveProgress) {
|
||||
this.saveProgress();
|
||||
}
|
||||
|
||||
// 进度回调
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
current: overallIndex,
|
||||
total: skus.length,
|
||||
completed: this.progress.completed,
|
||||
failed: this.progress.failed,
|
||||
lastResult: result
|
||||
});
|
||||
}
|
||||
|
||||
// SKU间延迟
|
||||
if (i < pendingSkus.length - 1) {
|
||||
await this.delay(this.config.skuDelay);
|
||||
}
|
||||
}
|
||||
|
||||
this.progress.endTime = Date.now();
|
||||
this.progress.totalDuration = (this.progress.endTime - this.progress.startTime) / 1000;
|
||||
|
||||
console.log(`\n${'='.repeat(60)}`);
|
||||
console.log(`Batch processing complete!`);
|
||||
console.log(`Total: ${skus.length}, Success: ${this.progress.completed}, Failed: ${this.progress.failed}`);
|
||||
console.log(`Duration: ${this.progress.totalDuration.toFixed(1)}s`);
|
||||
console.log(`${'='.repeat(60)}\n`);
|
||||
|
||||
return {
|
||||
total: skus.length,
|
||||
completed: this.progress.completed,
|
||||
failed: this.progress.failed,
|
||||
duration: this.progress.totalDuration,
|
||||
results: this.progress.results
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 延迟函数
|
||||
* @param {number} ms - 毫秒数
|
||||
*/
|
||||
delay(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存进度到文件
|
||||
*/
|
||||
saveProgress() {
|
||||
try {
|
||||
const progressPath = path.join(process.cwd(), this.config.progressFile);
|
||||
fs.writeFileSync(progressPath, JSON.stringify(this.progress.results, null, 2));
|
||||
} catch (error) {
|
||||
console.error('Failed to save progress:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载之前的进度
|
||||
* @returns {array} 之前的处理结果
|
||||
*/
|
||||
loadProgress() {
|
||||
try {
|
||||
const progressPath = path.join(process.cwd(), this.config.progressFile);
|
||||
if (fs.existsSync(progressPath)) {
|
||||
const content = fs.readFileSync(progressPath, 'utf-8');
|
||||
return JSON.parse(content);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load progress:', error.message);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* 清除进度文件
|
||||
*/
|
||||
clearProgress() {
|
||||
try {
|
||||
const progressPath = path.join(process.cwd(), this.config.progressFile);
|
||||
if (fs.existsSync(progressPath)) {
|
||||
fs.unlinkSync(progressPath);
|
||||
console.log('Progress cleared');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to clear progress:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成批处理报告
|
||||
* @returns {object} 报告数据
|
||||
*/
|
||||
generateReport() {
|
||||
const successResults = this.progress.results.filter(r => r.status === 'success');
|
||||
const failedResults = this.progress.results.filter(r => r.status === 'failed');
|
||||
|
||||
const totalImages = successResults.reduce((sum, r) => {
|
||||
return sum + (r.summary?.success || 0);
|
||||
}, 0);
|
||||
|
||||
return {
|
||||
summary: {
|
||||
total_skus: this.progress.total,
|
||||
successful_skus: this.progress.completed,
|
||||
failed_skus: this.progress.failed,
|
||||
success_rate: ((this.progress.completed / this.progress.total) * 100).toFixed(1) + '%',
|
||||
total_images_generated: totalImages,
|
||||
total_duration: this.progress.totalDuration?.toFixed(1) + 's',
|
||||
average_time_per_sku: (this.progress.totalDuration / this.progress.total).toFixed(1) + 's'
|
||||
},
|
||||
failed_skus: failedResults.map(r => ({
|
||||
sku_id: r.sku_id,
|
||||
error: r.error
|
||||
})),
|
||||
successful_skus: successResults.map(r => ({
|
||||
sku_id: r.sku_id,
|
||||
images: r.summary
|
||||
}))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从文件加载SKU列表
|
||||
* @param {string} filePath - 文件路径（JSON或CSV）
|
||||
* @returns {array} SKU数组
|
||||
*/
|
||||
function loadSKUsFromFile(filePath) {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
|
||||
if (ext === '.json') {
|
||||
const data = JSON.parse(content);
|
||||
return Array.isArray(data) ? data : [data];
|
||||
}
|
||||
|
||||
// 简单CSV解析（假设第一行是表头）
|
||||
if (ext === '.csv') {
|
||||
const lines = content.split('\n').filter(line => line.trim());
|
||||
const headers = lines[0].split(',').map(h => h.trim());
|
||||
|
||||
return lines.slice(1).map(line => {
|
||||
const values = line.split(',').map(v => v.trim());
|
||||
const obj = {};
|
||||
headers.forEach((h, i) => {
|
||||
obj[h] = values[i];
|
||||
});
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error('Unsupported file format. Use .json or .csv');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
BatchProcessor,
|
||||
loadSKUsFromFile,
|
||||
DEFAULT_CONFIG
|
||||
};
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user