wip

app/Traits/UpdateHelper.php

@@ -0,0 +1,488 @@
+<?php
+
+namespace App\Traits;
+
+use App\Models\SofaScorePlayerPerSeason;
+use App\Models\StatistikoDetail;
+use App\Models\StatistikoPlayer;
+use App\Utilities\Notice;
+use Illuminate\Support\Facades\Concurrency;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Str;
+use PDO;
+use PDOStatement;
+
+use function Laravel\Prompts\select;
+
+trait UpdateHelper
+{
+    protected float $benchmarkStartTime;
+
+    protected int $benchmarkStartMemory;
+
+    protected int $startRowCount;
+
+    protected int $startQueries;
+
+    protected int $debug;
+
+    public function handle(): void
+    {
+        error_reporting(E_ALL);
+        ini_set('display_errors', 1);
+
+        $tmp = $this->options();
+        $this->debug = $tmp['d'] ?? 0;
+
+        if ($this->debug) {
+            Notice::note('Update Player Season', 'g');
+        }
+
+        if ($this->debug) {
+            $playerCount = $this->selectPlayer();
+        } else {
+            $playerCount = 1000000;
+        }
+
+        if ($this->debug) {
+            $this->startBenchmark();
+        }
+
+        try {
+            $this->updatePlayers($playerCount);
+        } catch (\Exception $e) {
+            $this->error(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value());
+        }
+
+        if ($this->debug) {
+            $this->endBenchmark();
+        }
+    }
+
+    // Flight::upsert([
+    //     ['departure' => 'Oakland', 'destination' => 'San Diego', 'price' => 99],
+    //     ['departure' => 'Chicago', 'destination' => 'New York', 'price' => 150]
+    // ], uniqueBy: ['departure', 'destination'], update: ['price']);
+
+    protected function selectPlayer(): string
+    {
+        $file = select(
+            label: 'How many players to update?',
+            options: ['100 Player', '1k Player', '10k Player', '100k Player', '1m Player']
+        );
+
+        return match ($file) {
+            '100 Player' => 100,
+            '1k Player' => 1000,
+            '10k Player' => 10000,
+            '100k Player' => 100000,
+            '1m Player' => 1000000,
+        };
+    }
+
+    protected function startBenchmark(string $table = 'sofa_score_player_per_seasons'): void
+    {
+        $this->startRowCount = DB::table($table)->count();
+        $this->benchmarkStartTime = microtime(true);
+        $this->benchmarkStartMemory = memory_get_usage();
+        DB::enableQueryLog();
+        $this->startQueries = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value;
+    }
+
+    protected function endBenchmark(string $table = 'sofa_score_player_per_seasons'): void
+    {
+        $executionTime = microtime(true) - $this->benchmarkStartTime;
+        $memoryUsage = round((memory_get_usage() - $this->benchmarkStartMemory) / 1024 / 1024, 2);
+        $queriesCount = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value - $this->startQueries - 1; // Subtract the Questions query itself
+
+        // Get row count after we've stopped tracking queries
+        $rowDiff = DB::table($table)->count() - $this->startRowCount;
+
+        $formattedTime = match (true) {
+            $executionTime >= 60 => sprintf('%dm %ds', floor($executionTime / 60), $executionTime % 60),
+            $executionTime >= 1 => round($executionTime, 2).'s',
+            default => round($executionTime * 1000).'ms',
+        };
+
+        $this->newLine();
+        $this->line(sprintf(
+            '⚡ <bg=bright-cyan;fg=black> TIME: %s </> <bg=bright-green;fg=black> MEM: %sMB </> <bg=bright-yellow;fg=black> SQL: %s </> <bg=bright-magenta;fg=black> ROWS: %s </>',
+            $formattedTime,
+            $memoryUsage,
+            number_format($queriesCount),
+            number_format($rowDiff)
+        ));
+        $this->newLine();
+    }
+
+    protected function updatePlayers(int $playerCount)
+    {
+        $res_player = StatistikoDetail::query()->where('is_active', 1)->limit($playerCount)->pluck('ss_player_id')->toArray();
+
+        $chunks = array_chunk($res_player, 1000);
+
+        foreach ($chunks as $chunk) {
+
+            $res = StatistikoPlayer::with('event')->with('details')->whereIn('ss_player_id', $chunk)->orderBy('ss_player_id')->get();
+
+            $collection = [];
+            foreach ($res as $val) {
+
+                $ss_player_id = (int) $val->ss_player_id;
+                $activeteam_id = (int) $val->activeteam_id;
+                $minutes_played = (int) $val->minutes_played;
+                $ss_team_id = (int) $val->details->ss_team_id;
+                $season = $val->event->season;
+                $home_id = (int) $val->event->home_id;
+                $away_id = (int) $val->event->away_id;
+
+                if (! isset($collection[$season][$ss_player_id])) {
+                    $this->array_template($collection, $ss_player_id, $season);
+                }
+                if (! isset($collection['total'][$ss_player_id])) {
+                    $this->array_template($collection, $ss_player_id, 'total');
+                }
+
+                if ($minutes_played > 0) {
+                    if ($activeteam_id == $home_id) {
+                        $collection[$season][$ss_player_id]['events_played_home'] += 1;
+                        $collection[$season][$ss_player_id]['minutes_played_home'] += $minutes_played;
+                        $collection['total'][$ss_player_id]['events_played_home'] += 1;
+                        $collection['total'][$ss_player_id]['minutes_played_home'] += $minutes_played;
+                    } elseif ($activeteam_id == $away_id) {
+                        $collection[$season][$ss_player_id]['events_played_away'] += 1;
+                        $collection[$season][$ss_player_id]['minutes_played_away'] += $minutes_played;
+                        $collection['total'][$ss_player_id]['minutes_played_away'] += 1;
+                        $collection['total'][$ss_player_id]['minutes_played_away'] += $minutes_played;
+                    }
+
+                    $cards = $val->card_yellow + $val->card_yellowred + $val->card_red;
+
+                    $collection[$season][$ss_player_id]['ss_player_id'] = $ss_player_id;
+                    $collection[$season][$ss_player_id]['ss_team_id'] = $ss_team_id;
+                    $collection[$season][$ss_player_id]['season'] = $season;
+                    $collection[$season][$ss_player_id]['total_events']++;
+                    $collection[$season][$ss_player_id]['total_min'] += $minutes_played;
+                    $collection[$season][$ss_player_id]['average_min'] = round($collection[$season][$ss_player_id]['total_min'] / $collection[$season][$ss_player_id]['total_events']);
+                    $collection[$season][$ss_player_id]['goals'] += $val->goals;
+                    $collection[$season][$ss_player_id]['fouls'] += $val->fouls;
+                    $collection[$season][$ss_player_id]['cards'] += $cards;
+                    $collection[$season][$ss_player_id]['offsides'] += $val->offsides;
+                    $collection[$season][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
+                    $collection[$season][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
+
+                    $collection['total'][$ss_player_id]['ss_player_id'] = $ss_player_id;
+                    $collection['total'][$ss_player_id]['ss_team_id'] = $ss_team_id;
+                    $collection['total'][$ss_player_id]['season'] = 'total';
+                    $collection['total'][$ss_player_id]['total_events']++;
+                    $collection['total'][$ss_player_id]['total_min'] += $minutes_played;
+                    $collection['total'][$ss_player_id]['average_min'] = round($collection['total'][$ss_player_id]['total_min'] / $collection['total'][$ss_player_id]['total_events']);
+                    $collection['total'][$ss_player_id]['goals'] += $val->goals;
+                    $collection['total'][$ss_player_id]['fouls'] += $val->fouls;
+                    $collection['total'][$ss_player_id]['cards'] += $cards;
+                    $collection['total'][$ss_player_id]['offsides'] += $val->offsides;
+                    $collection['total'][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
+                    $collection['total'][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
+                }
+            }
+
+            $collection_1d = [];
+            foreach ($collection as $arr) {
+                foreach ($arr as $row) {
+                    $collection_1d[] = $row;
+                }
+            }
+            SofaScorePlayerPerSeason::upsert(
+                $collection_1d,
+                [
+                    'ss_player_id',
+                    'ss_team_id',
+                    'season',
+                ],
+                [
+                    'total_events',
+                    'total_min',
+                    'average_min',
+                    'events_played_home',
+                    'events_played_away',
+                    'minutes_played_home',
+                    'minutes_played_away',
+                    'goals',
+                    'fouls',
+                    'offsides',
+                    'shots_on_target',
+                    'shots_off_target',
+                    'cards',
+                ]
+            );
+        }
+    }
+
+    private function array_template(array &$collection, int $ss_player_id, string $season)
+    {
+        $collection[$season][$ss_player_id]['ss_player_id'] = 0;
+        $collection[$season][$ss_player_id]['ss_team_id'] = 0;
+        $collection[$season][$ss_player_id]['season'] = '';
+        $collection[$season][$ss_player_id]['total_events'] = 0;
+        $collection[$season][$ss_player_id]['total_min'] = 0;
+        $collection[$season][$ss_player_id]['average_min'] = 0;
+        $collection[$season][$ss_player_id]['events_played_home'] = 0;
+        $collection[$season][$ss_player_id]['events_played_away'] = 0;
+        $collection[$season][$ss_player_id]['minutes_played_home'] = 0;
+        $collection[$season][$ss_player_id]['minutes_played_away'] = 0;
+        $collection[$season][$ss_player_id]['goals'] = 0;
+        $collection[$season][$ss_player_id]['fouls'] = 0;
+        $collection[$season][$ss_player_id]['offsides'] = 0;
+        $collection[$season][$ss_player_id]['shots_on_target'] = 0;
+        $collection[$season][$ss_player_id]['shots_off_target'] = 0;
+        $collection[$season][$ss_player_id]['cards'] = 0;
+    }
+
+    private function import08ManualStreamingWithPdo(string $filePath): void
+    {
+        // 100 7ms / 0MB
+        // 1K 78ms / 0MB
+        // 10K 328ms / 0MB
+        // 100K 2.9s / 0MB
+        // 1M 28s / 0MB
+        $data = [];
+        $handle = fopen($filePath, 'rb');
+        fgetcsv($handle); // skip header
+        $now = now()->format('Y-m-d H:i:s');
+        $pdo = DB::connection()->getPdo();
+
+        while (($row = fgetcsv($handle)) !== false) {
+            $data[] = [
+                'custom_id' => $row[0],
+                'name' => $row[1],
+                'email' => $row[2],
+                'company' => $row[3],
+                'city' => $row[4],
+                'country' => $row[5],
+                'birthday' => $row[6],
+                'created_at' => $now,
+                'updated_at' => $now,
+            ];
+
+            if (count($data) === 1000) {
+                // Build the SQL query for the chunk
+                $columns = array_keys($data[0]);
+                $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
+
+                $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
+
+                // Flatten the data array for the query
+                $values = [];
+                foreach ($data as $row) {
+                    $values = array_merge($values, array_values($row));
+                }
+
+                $pdo->prepare($sql)->execute($values);
+                $data = [];
+            }
+        }
+
+        if (! empty($data)) {
+            $columns = array_keys($data[0]);
+            $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
+
+            $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
+
+            $values = [];
+            foreach ($data as $row) {
+                $values = array_merge($values, array_values($row));
+            }
+
+            $pdo->prepare($sql)->execute($values);
+        }
+
+        fclose($handle);
+    }
+
+    private function import09PDOPrepared(string $filePath): void
+    {
+        // Direct database connection with prepared statements
+        // 100 41ms / 0 MB
+        // 1K 237ms /
+        // 10K 2.21s
+        // 100K 25.27s
+        // 1M 4m43s
+        // 2M
+        $now = now()->format('Y-m-d H:i:s');
+        $handle = fopen($filePath, 'r');
+        fgets($handle); // skip header
+
+        try {
+            $pdo = DB::connection()->getPdo();
+            $stmt = $pdo->prepare('
+            INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ');
+
+            while (($row = fgetcsv($handle)) !== false) {
+                $stmt->execute([
+                    $row[0],
+                    $row[1],
+                    $row[2],
+                    $row[3],
+                    $row[4],
+                    $row[5],
+                    $row[6],
+                    $now,
+                    $now,
+                ]);
+            }
+        } finally {
+            fclose($handle);
+        }
+    }
+
+    private function import10PDOPreparedChunked(string $filePath): void
+    {
+        // Direct database connection with prepared statements
+        // 100 12ms / 0.15MB
+        // 1K 49ms / 0.74MB
+        // 10K 222ms / 0.74MB
+        // 100K 1.5s / 0.74MB
+        // 1M 15.3s / 0.74MB
+        // 2M 24s / 0.74MN
+        $now = now()->format('Y-m-d H:i:s');
+        $handle = fopen($filePath, 'r');
+        fgetcsv($handle); // skip header
+        $chunkSize = 500;
+        $chunks = [];
+
+        try {
+            $stmt = $this->prepareChunkedStatement($chunkSize);
+
+            while (($row = fgetcsv($handle)) !== false) {
+                $chunks = array_merge($chunks, [
+                    $row[0], $row[1], $row[2], $row[3], $row[4],
+                    $row[5], $row[6], $now, $now,
+                ]);
+
+                if (count($chunks) === $chunkSize * 9) {  // 9 columns
+                    $stmt->execute($chunks);
+                    $chunks = [];
+                }
+            }
+
+            // Handle remaining records
+            if (! empty($chunks)) {
+                $remainingRows = count($chunks) / 9;
+                $stmt = $this->prepareChunkedStatement($remainingRows);
+                $stmt->execute($chunks);
+            }
+        } finally {
+            fclose($handle);
+        }
+    }
+
+    private function import11Concurrent(string $filePath): void
+    {
+        // 100 168ms
+        // 1K 172ms
+        // 10K 234ms
+        // 100K 595ms
+        // 1M 4.36s
+        // 2M 8.8s
+        $now = now()->format('Y-m-d H:i:s');
+        $numberOfProcesses = 10;
+
+        $tasks = [];
+        for ($i = 0; $i < $numberOfProcesses; $i++) {
+            $tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
+                DB::reconnect();
+
+                $handle = fopen($filePath, 'r');
+                fgets($handle); // Skip header
+                $currentLine = 0;
+                $customers = [];
+
+                while (($line = fgets($handle)) !== false) {
+                    // Each process takes every Nth line
+                    if ($currentLine++ % $numberOfProcesses !== $i) {
+                        continue;
+                    }
+
+                    $row = str_getcsv($line);
+                    $customers[] = [
+                        'custom_id' => $row[0],
+                        'name' => $row[1],
+                        'email' => $row[2],
+                        'company' => $row[3],
+                        'city' => $row[4],
+                        'country' => $row[5],
+                        'birthday' => $row[6],
+                        'created_at' => $now,
+                        'updated_at' => $now,
+                    ];
+
+                    if (count($customers) === 1000) {
+                        DB::table('customers')->insert($customers);
+                        $customers = [];
+                    }
+                }
+
+                if (! empty($customers)) {
+                    DB::table('customers')->insert($customers);
+                }
+
+                fclose($handle);
+
+                return true;
+            };
+        }
+
+        Concurrency::run($tasks);
+
+    }
+
+    private function import12LoadDataInfile(string $filePath): void
+    {
+        // MySQL specific, fastest approach
+        // 100 10ms / 0MB
+        // 1K 29ms / 0MB
+        // 10K 115ms / 0MB
+        // 100K 567ms / 0MB
+        // 1M 5s / 0MB
+        // 2M 11s / 0MB
+        $pdo = DB::connection()->getPdo();
+        $pdo->setAttribute(PDO::MYSQL_ATTR_LOCAL_INFILE, true);
+
+        $filepath = str_replace('\\', '/', $filePath);
+
+        $query = <<<SQL
+    LOAD DATA LOCAL INFILE '$filepath'
+    INTO TABLE customers
+    FIELDS TERMINATED BY ','
+    ENCLOSED BY '"'
+    LINES TERMINATED BY '\n'
+    IGNORE 1 LINES
+    (@col1, @col2, @col3, @col4, @col5, @col6, @col7)
+    SET
+        custom_id = @col1,
+        name = @col2,
+        email = @col3,
+        company = @col4,
+        city = @col5,
+        country = @col6,
+        birthday = @col7,
+        created_at = NOW(),
+        updated_at = NOW()
+    SQL;
+
+        $pdo->exec($query);
+    }
+
+    private function prepareChunkedStatement($chunkSize): PDOStatement
+    {
+        $rowPlaceholders = '(?, ?, ?, ?, ?, ?, ?, ?, ?)';
+        $placeholders = implode(',', array_fill(0, $chunkSize, $rowPlaceholders));
+
+        return DB::connection()->getPdo()->prepare("
+        INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
+        VALUES {$placeholders}
+    ");
+    }
+}