options(); $this->debug = $tmp['d'] ?? 0; if ($this->debug) { Notice::note('Update Player Season', 'g'); } if ($this->debug) { $playerCount = $this->selectPlayer(); } else { $playerCount = 1000000; } if ($this->debug) { $this->startBenchmark(); } try { $this->updatePlayers($playerCount); } catch (\Exception $e) { $this->error(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value()); } if ($this->debug) { $this->endBenchmark(); } } // Flight::upsert([ // ['departure' => 'Oakland', 'destination' => 'San Diego', 'price' => 99], // ['departure' => 'Chicago', 'destination' => 'New York', 'price' => 150] // ], uniqueBy: ['departure', 'destination'], update: ['price']); protected function selectPlayer(): string { $file = select( label: 'How many players to update?', options: ['100 Player', '1k Player', '10k Player', '100k Player', '1m Player'] ); return match ($file) { '100 Player' => 100, '1k Player' => 1000, '10k Player' => 10000, '100k Player' => 100000, '1m Player' => 1000000, }; } protected function startBenchmark(string $table = 'sofa_score_player_per_seasons'): void { $this->startRowCount = DB::table($table)->count(); $this->benchmarkStartTime = microtime(true); $this->benchmarkStartMemory = memory_get_usage(); DB::enableQueryLog(); $this->startQueries = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value; } protected function endBenchmark(string $table = 'sofa_score_player_per_seasons'): void { $executionTime = microtime(true) - $this->benchmarkStartTime; $memoryUsage = round((memory_get_usage() - $this->benchmarkStartMemory) / 1024 / 1024, 2); $queriesCount = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value - $this->startQueries - 1; // Subtract the Questions query itself // Get row count after we've stopped tracking queries $rowDiff = DB::table($table)->count() - $this->startRowCount; $formattedTime = match (true) { $executionTime >= 60 => sprintf('%dm %ds', floor($executionTime / 60), $executionTime % 60), $executionTime >= 1 => round($executionTime, 2).'s', default => round($executionTime * 1000).'ms', }; $this->newLine(); $this->line(sprintf( '⚡ TIME: %s MEM: %sMB SQL: %s ROWS: %s ', $formattedTime, $memoryUsage, number_format($queriesCount), number_format($rowDiff) )); $this->newLine(); } protected function updatePlayers(int $playerCount) { $res_player = StatistikoDetail::query()->where('is_active', 1)->limit($playerCount)->pluck('ss_player_id')->toArray(); $chunks = array_chunk($res_player, 1000); foreach ($chunks as $chunk) { $res = StatistikoPlayer::with('event')->with('details')->whereIn('ss_player_id', $chunk)->orderBy('ss_player_id')->get(); $collection = []; foreach ($res as $val) { $ss_player_id = (int) $val->ss_player_id; $activeteam_id = (int) $val->activeteam_id; $minutes_played = (int) $val->minutes_played; $ss_team_id = (int) $val->details->ss_team_id; $season = $val->event->season; $home_id = (int) $val->event->home_id; $away_id = (int) $val->event->away_id; if ($season) { if (! isset($collection[$season][$ss_player_id])) { $this->array_template($collection, $ss_player_id, $season); } if (! isset($collection['total'][$ss_player_id])) { $this->array_template($collection, $ss_player_id, 'total'); } if ($minutes_played > 0) { if ($activeteam_id == $home_id) { $collection[$season][$ss_player_id]['events_played_home'] += 1; $collection[$season][$ss_player_id]['minutes_played_home'] += $minutes_played; $collection['total'][$ss_player_id]['events_played_home'] += 1; $collection['total'][$ss_player_id]['minutes_played_home'] += $minutes_played; } elseif ($activeteam_id == $away_id) { $collection[$season][$ss_player_id]['events_played_away'] += 1; $collection[$season][$ss_player_id]['minutes_played_away'] += $minutes_played; $collection['total'][$ss_player_id]['minutes_played_away'] += 1; $collection['total'][$ss_player_id]['minutes_played_away'] += $minutes_played; } $cards = $val->card_yellow + $val->card_yellowred + $val->card_red; $collection[$season][$ss_player_id]['ss_player_id'] = $ss_player_id; $collection[$season][$ss_player_id]['ss_team_id'] = $ss_team_id; $collection[$season][$ss_player_id]['season'] = $season; $collection[$season][$ss_player_id]['total_events']++; $collection[$season][$ss_player_id]['total_min'] += $minutes_played; $collection[$season][$ss_player_id]['average_min'] = round($collection[$season][$ss_player_id]['total_min'] / $collection[$season][$ss_player_id]['total_events']); $collection[$season][$ss_player_id]['goals'] += $val->goals; $collection[$season][$ss_player_id]['fouls'] += $val->fouls; $collection[$season][$ss_player_id]['cards'] += $cards; $collection[$season][$ss_player_id]['offsides'] += $val->offsides; $collection[$season][$ss_player_id]['shots_on_target'] += $val->shots_on_target; $collection[$season][$ss_player_id]['shots_off_target'] += $val->shots_off_target; $collection['total'][$ss_player_id]['ss_player_id'] = $ss_player_id; $collection['total'][$ss_player_id]['ss_team_id'] = $ss_team_id; $collection['total'][$ss_player_id]['season'] = 'total'; $collection['total'][$ss_player_id]['total_events']++; $collection['total'][$ss_player_id]['total_min'] += $minutes_played; $collection['total'][$ss_player_id]['average_min'] = round($collection['total'][$ss_player_id]['total_min'] / $collection['total'][$ss_player_id]['total_events']); $collection['total'][$ss_player_id]['goals'] += $val->goals; $collection['total'][$ss_player_id]['fouls'] += $val->fouls; $collection['total'][$ss_player_id]['cards'] += $cards; $collection['total'][$ss_player_id]['offsides'] += $val->offsides; $collection['total'][$ss_player_id]['shots_on_target'] += $val->shots_on_target; $collection['total'][$ss_player_id]['shots_off_target'] += $val->shots_off_target; } } } $collection_1d = []; foreach ($collection as $arr) { foreach ($arr as $row) { $collection_1d[] = $row; } } SofaScorePlayerPerSeason::upsert( $collection_1d, [ 'ss_player_id', 'ss_team_id', 'season', ], [ 'total_events', 'total_min', 'average_min', 'events_played_home', 'events_played_away', 'minutes_played_home', 'minutes_played_away', 'goals', 'fouls', 'offsides', 'shots_on_target', 'shots_off_target', 'cards', ] ); } } private function array_template(array &$collection, int $ss_player_id, string $season) { $collection[$season][$ss_player_id]['ss_player_id'] = 0; $collection[$season][$ss_player_id]['ss_team_id'] = 0; $collection[$season][$ss_player_id]['season'] = ''; $collection[$season][$ss_player_id]['total_events'] = 0; $collection[$season][$ss_player_id]['total_min'] = 0; $collection[$season][$ss_player_id]['average_min'] = 0; $collection[$season][$ss_player_id]['events_played_home'] = 0; $collection[$season][$ss_player_id]['events_played_away'] = 0; $collection[$season][$ss_player_id]['minutes_played_home'] = 0; $collection[$season][$ss_player_id]['minutes_played_away'] = 0; $collection[$season][$ss_player_id]['goals'] = 0; $collection[$season][$ss_player_id]['fouls'] = 0; $collection[$season][$ss_player_id]['offsides'] = 0; $collection[$season][$ss_player_id]['shots_on_target'] = 0; $collection[$season][$ss_player_id]['shots_off_target'] = 0; $collection[$season][$ss_player_id]['cards'] = 0; } private function import08ManualStreamingWithPdo(string $filePath): void { // 100 7ms / 0MB // 1K 78ms / 0MB // 10K 328ms / 0MB // 100K 2.9s / 0MB // 1M 28s / 0MB $data = []; $handle = fopen($filePath, 'rb'); fgetcsv($handle); // skip header $now = now()->format('Y-m-d H:i:s'); $pdo = DB::connection()->getPdo(); while (($row = fgetcsv($handle)) !== false) { $data[] = [ 'custom_id' => $row[0], 'name' => $row[1], 'email' => $row[2], 'company' => $row[3], 'city' => $row[4], 'country' => $row[5], 'birthday' => $row[6], 'created_at' => $now, 'updated_at' => $now, ]; if (count($data) === 1000) { // Build the SQL query for the chunk $columns = array_keys($data[0]); $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ','); $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders; // Flatten the data array for the query $values = []; foreach ($data as $row) { $values = array_merge($values, array_values($row)); } $pdo->prepare($sql)->execute($values); $data = []; } } if (! empty($data)) { $columns = array_keys($data[0]); $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ','); $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders; $values = []; foreach ($data as $row) { $values = array_merge($values, array_values($row)); } $pdo->prepare($sql)->execute($values); } fclose($handle); } private function import09PDOPrepared(string $filePath): void { // Direct database connection with prepared statements // 100 41ms / 0 MB // 1K 237ms / // 10K 2.21s // 100K 25.27s // 1M 4m43s // 2M $now = now()->format('Y-m-d H:i:s'); $handle = fopen($filePath, 'r'); fgets($handle); // skip header try { $pdo = DB::connection()->getPdo(); $stmt = $pdo->prepare(' INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) '); while (($row = fgetcsv($handle)) !== false) { $stmt->execute([ $row[0], $row[1], $row[2], $row[3], $row[4], $row[5], $row[6], $now, $now, ]); } } finally { fclose($handle); } } private function import10PDOPreparedChunked(string $filePath): void { // Direct database connection with prepared statements // 100 12ms / 0.15MB // 1K 49ms / 0.74MB // 10K 222ms / 0.74MB // 100K 1.5s / 0.74MB // 1M 15.3s / 0.74MB // 2M 24s / 0.74MN $now = now()->format('Y-m-d H:i:s'); $handle = fopen($filePath, 'r'); fgetcsv($handle); // skip header $chunkSize = 500; $chunks = []; try { $stmt = $this->prepareChunkedStatement($chunkSize); while (($row = fgetcsv($handle)) !== false) { $chunks = array_merge($chunks, [ $row[0], $row[1], $row[2], $row[3], $row[4], $row[5], $row[6], $now, $now, ]); if (count($chunks) === $chunkSize * 9) { // 9 columns $stmt->execute($chunks); $chunks = []; } } // Handle remaining records if (! empty($chunks)) { $remainingRows = count($chunks) / 9; $stmt = $this->prepareChunkedStatement($remainingRows); $stmt->execute($chunks); } } finally { fclose($handle); } } private function import11Concurrent(string $filePath): void { // 100 168ms // 1K 172ms // 10K 234ms // 100K 595ms // 1M 4.36s // 2M 8.8s $now = now()->format('Y-m-d H:i:s'); $numberOfProcesses = 10; $tasks = []; for ($i = 0; $i < $numberOfProcesses; $i++) { $tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) { DB::reconnect(); $handle = fopen($filePath, 'r'); fgets($handle); // Skip header $currentLine = 0; $customers = []; while (($line = fgets($handle)) !== false) { // Each process takes every Nth line if ($currentLine++ % $numberOfProcesses !== $i) { continue; } $row = str_getcsv($line); $customers[] = [ 'custom_id' => $row[0], 'name' => $row[1], 'email' => $row[2], 'company' => $row[3], 'city' => $row[4], 'country' => $row[5], 'birthday' => $row[6], 'created_at' => $now, 'updated_at' => $now, ]; if (count($customers) === 1000) { DB::table('customers')->insert($customers); $customers = []; } } if (! empty($customers)) { DB::table('customers')->insert($customers); } fclose($handle); return true; }; } Concurrency::run($tasks); } private function import12LoadDataInfile(string $filePath): void { // MySQL specific, fastest approach // 100 10ms / 0MB // 1K 29ms / 0MB // 10K 115ms / 0MB // 100K 567ms / 0MB // 1M 5s / 0MB // 2M 11s / 0MB $pdo = DB::connection()->getPdo(); $pdo->setAttribute(PDO::MYSQL_ATTR_LOCAL_INFILE, true); $filepath = str_replace('\\', '/', $filePath); $query = <<exec($query); } private function prepareChunkedStatement($chunkSize): PDOStatement { $rowPlaceholders = '(?, ?, ?, ?, ?, ?, ?, ?, ?)'; $placeholders = implode(',', array_fill(0, $chunkSize, $rowPlaceholders)); return DB::connection()->getPdo()->prepare(" INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at) VALUES {$placeholders} "); } }