wip SimpleExcelReader with LazyCollection
This commit is contained in:
salam committed
1 parent
baa1117f61
commit
048989fed8
6 files changed
+442
-200
No files matched your search
@@ -0,0 +1,189 @@
|
||||
<?php
|
||||
|
||||
namespace App\Filament\Imports;
|
||||
|
||||
use App\Models\StatistikoPlayer;
|
||||
use Filament\Notifications\Notification;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\HtmlString;
|
||||
use Illuminate\Support\LazyCollection;
|
||||
use Illuminate\Support\Str;
|
||||
use Spatie\SimpleExcel\SimpleExcelReader;
|
||||
|
||||
class StatistikoImporter
|
||||
{
|
||||
public static float $benchmarkStartTime;
|
||||
|
||||
public static int $benchmarkStartMemory;
|
||||
|
||||
public static int $startRowCount;
|
||||
|
||||
public static int $startQueries;
|
||||
|
||||
public static function run($mode, $record): void
|
||||
{
|
||||
error_reporting(E_ALL);
|
||||
|
||||
ini_set('display_errors', 1);
|
||||
|
||||
$table = $record['data_type'];
|
||||
|
||||
$filePath = storage_path().'/app/private/'.$record['file_name'];
|
||||
|
||||
self::startBenchmark($table);
|
||||
|
||||
try {
|
||||
self::handleImport($filePath, $table, $mode);
|
||||
} catch (\Exception $e) {
|
||||
dd(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value());
|
||||
}
|
||||
|
||||
self::endBenchmark();
|
||||
}
|
||||
|
||||
protected static function startBenchmark(string $table): void
|
||||
{
|
||||
self::$startRowCount = DB::connection('mongodb')->table($table)->count();
|
||||
self::$benchmarkStartTime = microtime(true);
|
||||
self::$benchmarkStartMemory = memory_get_usage();
|
||||
DB::connection('mongodb')->enableQueryLog();
|
||||
}
|
||||
|
||||
protected static function endBenchmark(string $table = 'customers'): void
|
||||
{
|
||||
$executionTime = microtime(true) - self::$benchmarkStartTime;
|
||||
$memoryUsage = round((memory_get_usage() - self::$benchmarkStartMemory) / 1024 / 1024, 2);
|
||||
|
||||
$logs = DB::connection('mongodb')->getQueryLog();
|
||||
$queriesCount = \count($logs);
|
||||
|
||||
// Get row count after we've stopped tracking queries
|
||||
$rowDiff = DB::connection('mongodb')->table($table)->count() - self::$startRowCount;
|
||||
|
||||
$formattedTime = match (true) {
|
||||
$executionTime >= 60 => sprintf('%dm %ds', floor($executionTime / 60), $executionTime % 60),
|
||||
$executionTime >= 1 => round($executionTime, 2).'s',
|
||||
default => round($executionTime * 1000).'ms',
|
||||
};
|
||||
|
||||
$str = '⚡ TIME: '.$formattedTime;
|
||||
$str .= '<br/>MEM: '.$memoryUsage.'MB';
|
||||
$str .= '<br/>SQL: '.number_format($queriesCount);
|
||||
$str .= '<br/>ROWS: '.number_format($rowDiff);
|
||||
|
||||
$recipient = auth()->user();
|
||||
Notification::make()
|
||||
->success()
|
||||
->title('Import compleated')
|
||||
->body(new HtmlString($str))
|
||||
->sendToDatabase($recipient);
|
||||
}
|
||||
|
||||
private static function handleImport(string $filePath, $table, $mode): void
|
||||
{
|
||||
$now = now()->format('Y-m-d H:i:s');
|
||||
|
||||
$chunkSize = 1000;
|
||||
|
||||
SimpleExcelReader::create($filePath)
|
||||
->getRows()
|
||||
->chunk($chunkSize)
|
||||
->each(function ($rowProperties) {
|
||||
dd($rowProperties);
|
||||
// processing rows
|
||||
});
|
||||
|
||||
// LazyCollection::make(function () use ($filePath) {
|
||||
// $handle = fopen($filePath, 'r');
|
||||
// fgets($handle); // skip header
|
||||
|
||||
// while (($line = fgets($handle)) !== false) {
|
||||
// yield str_getcsv($line);
|
||||
// }
|
||||
// fclose($handle);
|
||||
// })->each(function ($row) {
|
||||
// dd($row);
|
||||
// });
|
||||
// ->map(fn ($row) => [
|
||||
// 'custom_id' => $row[0],
|
||||
// 'name' => $row[1],
|
||||
// 'email' => $row[2],
|
||||
// 'company' => $row[3],
|
||||
// 'city' => $row[4],
|
||||
// 'country' => $row[5],
|
||||
// 'birthday' => $row[6],
|
||||
// 'created_at' => $now,
|
||||
// 'updated_at' => $now,
|
||||
// ])
|
||||
// ->chunk($chunkSize)
|
||||
// ->each(fn ($chunk) => $result = StatistikoPlayer::upsert(
|
||||
// [
|
||||
// ['title' => 'Inspector Maigret', 'recommended' => false, 'runtime' => 128],
|
||||
// ['title' => 'Petit Maman', 'recommended' => true, 'runtime' => 72],
|
||||
// ],
|
||||
// 'title',
|
||||
// 'recommended',
|
||||
// ));
|
||||
}
|
||||
|
||||
private static function import11Concurrent(string $filePath): void
|
||||
{
|
||||
// 100 168ms
|
||||
// 1K 172ms
|
||||
// 10K 234ms
|
||||
// 100K 595ms
|
||||
// 1M 4.36s
|
||||
// 2M 8.8s
|
||||
$now = now()->format('Y-m-d H:i:s');
|
||||
$numberOfProcesses = 10;
|
||||
|
||||
$tasks = [];
|
||||
for ($i = 0; $i < $numberOfProcesses; $i++) {
|
||||
$tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
|
||||
DB::reconnect();
|
||||
|
||||
$handle = fopen($filePath, 'r');
|
||||
fgets($handle); // Skip header
|
||||
$currentLine = 0;
|
||||
$customers = [];
|
||||
|
||||
while (($line = fgets($handle)) !== false) {
|
||||
// Each process takes every Nth line
|
||||
if ($currentLine++ % $numberOfProcesses !== $i) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$row = str_getcsv($line);
|
||||
$customers[] = [
|
||||
'custom_id' => $row[0],
|
||||
'name' => $row[1],
|
||||
'email' => $row[2],
|
||||
'company' => $row[3],
|
||||
'city' => $row[4],
|
||||
'country' => $row[5],
|
||||
'birthday' => $row[6],
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
];
|
||||
|
||||
if (count($customers) === 1000) {
|
||||
DB::table('customers')->insert($customers);
|
||||
$customers = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($customers)) {
|
||||
DB::table('customers')->insert($customers);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
|
||||
return true;
|
||||
};
|
||||
}
|
||||
|
||||
Concurrency::run($tasks);
|
||||
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user