getMessage())->limit(100)->value());
}
self::endBenchmark();
}
protected static function startBenchmark(string $table): void
{
self::$startRowCount = DB::connection('mongodb')->table($table)->count();
self::$benchmarkStartTime = microtime(true);
self::$benchmarkStartMemory = memory_get_usage();
DB::connection('mongodb')->enableQueryLog();
}
protected static function endBenchmark(string $table = 'customers'): void
{
$executionTime = microtime(true) - self::$benchmarkStartTime;
$memoryUsage = round((memory_get_usage() - self::$benchmarkStartMemory) / 1024 / 1024, 2);
$logs = DB::connection('mongodb')->getQueryLog();
$queriesCount = \count($logs);
// Get row count after we've stopped tracking queries
$rowDiff = DB::connection('mongodb')->table($table)->count() - self::$startRowCount;
$formattedTime = match (true) {
$executionTime >= 60 => sprintf('%dm %ds', floor($executionTime / 60), $executionTime % 60),
$executionTime >= 1 => round($executionTime, 2).'s',
default => round($executionTime * 1000).'ms',
};
$str = '⚡ TIME: '.$formattedTime;
$str .= '
MEM: '.$memoryUsage.'MB';
$str .= '
SQL: '.number_format($queriesCount);
$str .= '
ROWS: '.number_format($rowDiff);
$recipient = auth()->user();
Notification::make()
->success()
->title('Import compleated')
->body(new HtmlString($str))
->sendToDatabase($recipient);
}
private static function handleImport(string $filePath, $table, $mode): void
{
$now = now()->format('Y-m-d H:i:s');
$chunkSize = 1000;
SimpleExcelReader::create($filePath)
->getRows()
->chunk($chunkSize)
->each(function ($rowProperties) {
dd($rowProperties);
// processing rows
});
// LazyCollection::make(function () use ($filePath) {
// $handle = fopen($filePath, 'r');
// fgets($handle); // skip header
// while (($line = fgets($handle)) !== false) {
// yield str_getcsv($line);
// }
// fclose($handle);
// })->each(function ($row) {
// dd($row);
// });
// ->map(fn ($row) => [
// 'custom_id' => $row[0],
// 'name' => $row[1],
// 'email' => $row[2],
// 'company' => $row[3],
// 'city' => $row[4],
// 'country' => $row[5],
// 'birthday' => $row[6],
// 'created_at' => $now,
// 'updated_at' => $now,
// ])
// ->chunk($chunkSize)
// ->each(fn ($chunk) => $result = StatistikoPlayer::upsert(
// [
// ['title' => 'Inspector Maigret', 'recommended' => false, 'runtime' => 128],
// ['title' => 'Petit Maman', 'recommended' => true, 'runtime' => 72],
// ],
// 'title',
// 'recommended',
// ));
}
private static function import11Concurrent(string $filePath): void
{
// 100 168ms
// 1K 172ms
// 10K 234ms
// 100K 595ms
// 1M 4.36s
// 2M 8.8s
$now = now()->format('Y-m-d H:i:s');
$numberOfProcesses = 10;
$tasks = [];
for ($i = 0; $i < $numberOfProcesses; $i++) {
$tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
DB::reconnect();
$handle = fopen($filePath, 'r');
fgets($handle); // Skip header
$currentLine = 0;
$customers = [];
while (($line = fgets($handle)) !== false) {
// Each process takes every Nth line
if ($currentLine++ % $numberOfProcesses !== $i) {
continue;
}
$row = str_getcsv($line);
$customers[] = [
'custom_id' => $row[0],
'name' => $row[1],
'email' => $row[2],
'company' => $row[3],
'city' => $row[4],
'country' => $row[5],
'birthday' => $row[6],
'created_at' => $now,
'updated_at' => $now,
];
if (count($customers) === 1000) {
DB::table('customers')->insert($customers);
$customers = [];
}
}
if (! empty($customers)) {
DB::table('customers')->insert($customers);
}
fclose($handle);
return true;
};
}
Concurrency::run($tasks);
}
}