wip SimpleExcelReader with LazyCollection

app/Filament/Clusters/StaticData/Resources/StatistikoImports/Tables/StatistikoImportsTable.php

@@ -2,11 +2,13 @@
 
 namespace App\Filament\Clusters\StaticData\Resources\StatistikoImports\Tables;
 
+use App\Filament\Imports\StatistikoImporter;
 use Filament\Actions\Action;
 use Filament\Actions\ActionGroup;
 use Filament\Actions\BulkActionGroup;
 use Filament\Actions\DeleteAction;
 use Filament\Actions\DeleteBulkAction;
+use Filament\Notifications\Notification;
 use Filament\Tables\Columns\IconColumn;
 use Filament\Tables\Columns\SelectColumn;
 use Filament\Tables\Columns\TextColumn;
@@ -18,6 +20,7 @@ class StatistikoImportsTable
     public static function configure(Table $table): Table
     {
         return $table
+            ->striped()
             ->columns([
                 IconColumn::make('sport.icon')
                     ->label('Sport')
@@ -57,11 +60,25 @@ class StatistikoImportsTable
             ->recordActions([
                 ActionGroup::make([
                     Action::make('Import')
-                        ->label('Just Import (very fast)')
-                        ->icon('phosphor-database'),
+                        ->label('Import (fast)')
+                        ->icon('phosphor-database')
+                        ->action(function ($record) {
+                            StatistikoImporter::run('import', $record);
+                        })->successNotification(
+                            Notification::make()
+                                ->info()
+                                ->title('Import started')
+                        ),
                     Action::make('Update')
-                        ->label('Import and Update (slow)')
-                        ->icon('phosphor-database'),
+                        ->label('Update or Import (slow)')
+                        ->icon('phosphor-database')
+                        ->action(function ($record) {
+                            StatistikoImporter::run('update', $record);
+                        })->successNotification(
+                            Notification::make()
+                                ->info()
+                                ->title('Import & Update started')
+                        ),
                     DeleteAction::make()
                         ->label('Delete file'),
                 ]),

app/Filament/Imports/StatistikoImporter.php

@@ -0,0 +1,189 @@
+<?php
+
+namespace App\Filament\Imports;
+
+use App\Models\StatistikoPlayer;
+use Filament\Notifications\Notification;
+use Illuminate\Support\Facades\Concurrency;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\HtmlString;
+use Illuminate\Support\LazyCollection;
+use Illuminate\Support\Str;
+use Spatie\SimpleExcel\SimpleExcelReader;
+
+class StatistikoImporter
+{
+    public static float $benchmarkStartTime;
+
+    public static int $benchmarkStartMemory;
+
+    public static int $startRowCount;
+
+    public static int $startQueries;
+
+    public static function run($mode, $record): void
+    {
+        error_reporting(E_ALL);
+
+        ini_set('display_errors', 1);
+
+        $table = $record['data_type'];
+
+        $filePath = storage_path().'/app/private/'.$record['file_name'];
+
+        self::startBenchmark($table);
+
+        try {
+            self::handleImport($filePath, $table, $mode);
+        } catch (\Exception $e) {
+            dd(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value());
+        }
+
+        self::endBenchmark();
+    }
+
+    protected static function startBenchmark(string $table): void
+    {
+        self::$startRowCount = DB::connection('mongodb')->table($table)->count();
+        self::$benchmarkStartTime = microtime(true);
+        self::$benchmarkStartMemory = memory_get_usage();
+        DB::connection('mongodb')->enableQueryLog();
+    }
+
+    protected static function endBenchmark(string $table = 'customers'): void
+    {
+        $executionTime = microtime(true) - self::$benchmarkStartTime;
+        $memoryUsage = round((memory_get_usage() - self::$benchmarkStartMemory) / 1024 / 1024, 2);
+
+        $logs = DB::connection('mongodb')->getQueryLog();
+        $queriesCount = \count($logs);
+
+        // Get row count after we've stopped tracking queries
+        $rowDiff = DB::connection('mongodb')->table($table)->count() - self::$startRowCount;
+
+        $formattedTime = match (true) {
+            $executionTime >= 60 => sprintf('%dm %ds', floor($executionTime / 60), $executionTime % 60),
+            $executionTime >= 1 => round($executionTime, 2).'s',
+            default => round($executionTime * 1000).'ms',
+        };
+
+        $str = '⚡ TIME: '.$formattedTime;
+        $str .= '<br/>MEM: '.$memoryUsage.'MB';
+        $str .= '<br/>SQL: '.number_format($queriesCount);
+        $str .= '<br/>ROWS: '.number_format($rowDiff);
+
+        $recipient = auth()->user();
+        Notification::make()
+            ->success()
+            ->title('Import compleated')
+            ->body(new HtmlString($str))
+            ->sendToDatabase($recipient);
+    }
+
+    private static function handleImport(string $filePath, $table, $mode): void
+    {
+        $now = now()->format('Y-m-d H:i:s');
+
+        $chunkSize = 1000;
+
+        SimpleExcelReader::create($filePath)
+            ->getRows()
+            ->chunk($chunkSize)
+            ->each(function ($rowProperties) {
+                dd($rowProperties);
+                // processing rows
+            });
+
+        // LazyCollection::make(function () use ($filePath) {
+        //     $handle = fopen($filePath, 'r');
+        //     fgets($handle); // skip header
+
+        //     while (($line = fgets($handle)) !== false) {
+        //         yield str_getcsv($line);
+        //     }
+        //     fclose($handle);
+        // })->each(function ($row) {
+        //     dd($row);
+        // });
+        // ->map(fn ($row) => [
+        //     'custom_id' => $row[0],
+        //     'name' => $row[1],
+        //     'email' => $row[2],
+        //     'company' => $row[3],
+        //     'city' => $row[4],
+        //     'country' => $row[5],
+        //     'birthday' => $row[6],
+        //     'created_at' => $now,
+        //     'updated_at' => $now,
+        // ])
+        // ->chunk($chunkSize)
+        // ->each(fn ($chunk) => $result = StatistikoPlayer::upsert(
+        //     [
+        //         ['title' => 'Inspector Maigret', 'recommended' => false, 'runtime' => 128],
+        //         ['title' => 'Petit Maman', 'recommended' => true, 'runtime' => 72],
+        //     ],
+        //     'title',
+        //     'recommended',
+        // ));
+    }
+
+    private static function import11Concurrent(string $filePath): void
+    {
+        // 100 168ms
+        // 1K 172ms
+        // 10K 234ms
+        // 100K 595ms
+        // 1M 4.36s
+        // 2M 8.8s
+        $now = now()->format('Y-m-d H:i:s');
+        $numberOfProcesses = 10;
+
+        $tasks = [];
+        for ($i = 0; $i < $numberOfProcesses; $i++) {
+            $tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
+                DB::reconnect();
+
+                $handle = fopen($filePath, 'r');
+                fgets($handle); // Skip header
+                $currentLine = 0;
+                $customers = [];
+
+                while (($line = fgets($handle)) !== false) {
+                    // Each process takes every Nth line
+                    if ($currentLine++ % $numberOfProcesses !== $i) {
+                        continue;
+                    }
+
+                    $row = str_getcsv($line);
+                    $customers[] = [
+                        'custom_id' => $row[0],
+                        'name' => $row[1],
+                        'email' => $row[2],
+                        'company' => $row[3],
+                        'city' => $row[4],
+                        'country' => $row[5],
+                        'birthday' => $row[6],
+                        'created_at' => $now,
+                        'updated_at' => $now,
+                    ];
+
+                    if (count($customers) === 1000) {
+                        DB::table('customers')->insert($customers);
+                        $customers = [];
+                    }
+                }
+
+                if (! empty($customers)) {
+                    DB::table('customers')->insert($customers);
+                }
+
+                fclose($handle);
+
+                return true;
+            };
+        }
+
+        Concurrency::run($tasks);
+
+    }
+}

app/Models/StatistikoPlayer.php

@@ -13,18 +13,18 @@ class StatistikoPlayer extends Model
     protected $table = 'player';
 
     protected $fillable = [
-        'id_giocatore',
-        'giocatore',
-        'data_di_nascita',
-        'altezza',
-        'nazionalita',
-        'posizione',
+        'player_id',
+        'name',
+        'full_name',
+        'birthdate',
+        'height',
+        'nationality',
+        'position',
         'sport_radar_id',
         'id_squadra_tm',
-        'nome_completo',
-        'rigorista',
+        'rigorist',
         'national_team_id',
-        'data_aggiornamento',
+        'last_update',
         'league_code',
         'team_code',
         'team_name',
@@ -33,8 +33,8 @@ class StatistikoPlayer extends Model
     protected function casts(): array
     {
         return [
-            'data_di_nascita' => 'date',
-            'data_aggiornamento' => 'date',
+            'birthdate' => 'date',
+            'last_update' => 'date',
         ];
     }
 }

app/Providers/Filament/AdminPanelProvider.php

@@ -2,7 +2,7 @@
 
 namespace App\Providers\Filament;
 
-use Agencetwogether\HooksHelper\HooksHelperPlugin;
+// use Agencetwogether\HooksHelper\HooksHelperPlugin;
 use App\Filament\Exports\BetfairExporter;
 use App\Http\Middleware\RecordUserActivity;
 use Awcodes\LightSwitch\Enums\Alignment;
@@ -18,7 +18,7 @@ use Filament\Http\Middleware\Authenticate;
 use Filament\Http\Middleware\AuthenticateSession;
 use Filament\Http\Middleware\DisableBladeIconComponents;
 use Filament\Http\Middleware\DispatchServingFilamentEvent;
-use Filament\Pages\Dashboard;
+// use Filament\Pages\Dashboard;
 use Filament\Panel;
 use Filament\PanelProvider;
 use Filament\Support\Enums\Width;
@@ -65,7 +65,7 @@ class AdminPanelProvider extends PanelProvider
             ->emailVerification()
             ->emailChangeVerification()
             ->databaseNotifications()
-            // ->databaseNotificationsPolling('15s') default is 30
+            ->databaseNotificationsPolling('10s') // default is 30
             ->sidebarCollapsibleOnDesktop()
             ->profile(isSimple: false)
             ->default()