492 lines
18 KiB
PHP
492 lines
18 KiB
PHP
<?php
|
|
|
|
namespace App\Traits;
|
|
|
|
use App\Models\SofaScorePlayerPerSeason;
|
|
use App\Models\StatistikoDetail;
|
|
use App\Models\StatistikoPlayer;
|
|
use App\Utilities\Notice;
|
|
use Illuminate\Support\Facades\Concurrency;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Str;
|
|
use PDO;
|
|
use PDOStatement;
|
|
|
|
use function Laravel\Prompts\select;
|
|
|
|
trait UpdateHelper
|
|
{
|
|
protected float $benchmarkStartTime;
|
|
|
|
protected int $benchmarkStartMemory;
|
|
|
|
protected int $startRowCount;
|
|
|
|
protected int $startQueries;
|
|
|
|
protected int $debug;
|
|
|
|
public function handle(): void
|
|
{
|
|
error_reporting(E_ALL);
|
|
ini_set('display_errors', 1);
|
|
|
|
$tmp = $this->options();
|
|
$this->debug = $tmp['d'] ?? 0;
|
|
|
|
if ($this->debug) {
|
|
Notice::note('Update Player Season', 'g');
|
|
}
|
|
|
|
if ($this->debug) {
|
|
$playerCount = $this->selectPlayer();
|
|
} else {
|
|
$playerCount = 1000000;
|
|
}
|
|
|
|
if ($this->debug) {
|
|
$this->startBenchmark();
|
|
}
|
|
|
|
try {
|
|
$this->updatePlayers($playerCount);
|
|
} catch (\Exception $e) {
|
|
$this->error(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value());
|
|
}
|
|
|
|
if ($this->debug) {
|
|
$this->endBenchmark();
|
|
}
|
|
}
|
|
|
|
// Flight::upsert([
|
|
// ['departure' => 'Oakland', 'destination' => 'San Diego', 'price' => 99],
|
|
// ['departure' => 'Chicago', 'destination' => 'New York', 'price' => 150]
|
|
// ], uniqueBy: ['departure', 'destination'], update: ['price']);
|
|
|
|
protected function selectPlayer(): string
|
|
{
|
|
$file = select(
|
|
label: 'How many players to update?',
|
|
options: ['100 Player', '1k Player', '10k Player', '100k Player', '1m Player']
|
|
);
|
|
|
|
return match ($file) {
|
|
'100 Player' => 100,
|
|
'1k Player' => 1000,
|
|
'10k Player' => 10000,
|
|
'100k Player' => 100000,
|
|
'1m Player' => 1000000,
|
|
};
|
|
}
|
|
|
|
protected function startBenchmark(string $table = 'sofa_score_player_per_seasons'): void
|
|
{
|
|
$this->startRowCount = DB::table($table)->count();
|
|
$this->benchmarkStartTime = microtime(true);
|
|
$this->benchmarkStartMemory = memory_get_usage();
|
|
DB::enableQueryLog();
|
|
$this->startQueries = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value;
|
|
}
|
|
|
|
protected function endBenchmark(string $table = 'sofa_score_player_per_seasons'): void
|
|
{
|
|
$executionTime = microtime(true) - $this->benchmarkStartTime;
|
|
$memoryUsage = round((memory_get_usage() - $this->benchmarkStartMemory) / 1024 / 1024, 2);
|
|
$queriesCount = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value - $this->startQueries - 1; // Subtract the Questions query itself
|
|
|
|
// Get row count after we've stopped tracking queries
|
|
$rowDiff = DB::table($table)->count() - $this->startRowCount;
|
|
|
|
$formattedTime = match (true) {
|
|
$executionTime >= 60 => sprintf('%dm %ds', floor($executionTime / 60), $executionTime % 60),
|
|
$executionTime >= 1 => round($executionTime, 2).'s',
|
|
default => round($executionTime * 1000).'ms',
|
|
};
|
|
|
|
$this->newLine();
|
|
$this->line(sprintf(
|
|
'⚡ <bg=bright-cyan;fg=black> TIME: %s </> <bg=bright-green;fg=black> MEM: %sMB </> <bg=bright-yellow;fg=black> SQL: %s </> <bg=bright-magenta;fg=black> ROWS: %s </>',
|
|
$formattedTime,
|
|
$memoryUsage,
|
|
number_format($queriesCount),
|
|
number_format($rowDiff)
|
|
));
|
|
$this->newLine();
|
|
}
|
|
|
|
protected function updatePlayers(int $playerCount)
|
|
{
|
|
$res_player = StatistikoDetail::query()->where('is_active', 1)->limit($playerCount)->pluck('ss_player_id')->toArray();
|
|
|
|
$chunks = array_chunk($res_player, 1000);
|
|
|
|
foreach ($chunks as $chunk) {
|
|
|
|
$res = StatistikoPlayer::with('event')->with('details')->whereIn('ss_player_id', $chunk)->orderBy('ss_player_id')->get();
|
|
|
|
$collection = [];
|
|
foreach ($res as $val) {
|
|
|
|
$ss_player_id = (int) $val->ss_player_id;
|
|
$activeteam_id = (int) $val->activeteam_id;
|
|
$minutes_played = (int) $val->minutes_played;
|
|
$ss_team_id = (int) $val->details->ss_team_id;
|
|
$season = $val->event->season;
|
|
$home_id = (int) $val->event->home_id;
|
|
$away_id = (int) $val->event->away_id;
|
|
|
|
if ($season) {
|
|
|
|
if (! isset($collection[$season][$ss_player_id])) {
|
|
$this->array_template($collection, $ss_player_id, $season);
|
|
}
|
|
if (! isset($collection['total'][$ss_player_id])) {
|
|
$this->array_template($collection, $ss_player_id, 'total');
|
|
}
|
|
|
|
if ($minutes_played > 0) {
|
|
if ($activeteam_id == $home_id) {
|
|
$collection[$season][$ss_player_id]['events_played_home'] += 1;
|
|
$collection[$season][$ss_player_id]['minutes_played_home'] += $minutes_played;
|
|
$collection['total'][$ss_player_id]['events_played_home'] += 1;
|
|
$collection['total'][$ss_player_id]['minutes_played_home'] += $minutes_played;
|
|
} elseif ($activeteam_id == $away_id) {
|
|
$collection[$season][$ss_player_id]['events_played_away'] += 1;
|
|
$collection[$season][$ss_player_id]['minutes_played_away'] += $minutes_played;
|
|
$collection['total'][$ss_player_id]['minutes_played_away'] += 1;
|
|
$collection['total'][$ss_player_id]['minutes_played_away'] += $minutes_played;
|
|
}
|
|
|
|
$cards = $val->card_yellow + $val->card_yellowred + $val->card_red;
|
|
|
|
$collection[$season][$ss_player_id]['ss_player_id'] = $ss_player_id;
|
|
$collection[$season][$ss_player_id]['ss_team_id'] = $ss_team_id;
|
|
$collection[$season][$ss_player_id]['season'] = $season;
|
|
$collection[$season][$ss_player_id]['total_events']++;
|
|
$collection[$season][$ss_player_id]['total_min'] += $minutes_played;
|
|
$collection[$season][$ss_player_id]['average_min'] = round($collection[$season][$ss_player_id]['total_min'] / $collection[$season][$ss_player_id]['total_events']);
|
|
$collection[$season][$ss_player_id]['goals'] += $val->goals;
|
|
$collection[$season][$ss_player_id]['fouls'] += $val->fouls;
|
|
$collection[$season][$ss_player_id]['cards'] += $cards;
|
|
$collection[$season][$ss_player_id]['offsides'] += $val->offsides;
|
|
$collection[$season][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
|
|
$collection[$season][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
|
|
|
|
$collection['total'][$ss_player_id]['ss_player_id'] = $ss_player_id;
|
|
$collection['total'][$ss_player_id]['ss_team_id'] = $ss_team_id;
|
|
$collection['total'][$ss_player_id]['season'] = 'total';
|
|
$collection['total'][$ss_player_id]['total_events']++;
|
|
$collection['total'][$ss_player_id]['total_min'] += $minutes_played;
|
|
$collection['total'][$ss_player_id]['average_min'] = round($collection['total'][$ss_player_id]['total_min'] / $collection['total'][$ss_player_id]['total_events']);
|
|
$collection['total'][$ss_player_id]['goals'] += $val->goals;
|
|
$collection['total'][$ss_player_id]['fouls'] += $val->fouls;
|
|
$collection['total'][$ss_player_id]['cards'] += $cards;
|
|
$collection['total'][$ss_player_id]['offsides'] += $val->offsides;
|
|
$collection['total'][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
|
|
$collection['total'][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
|
|
}
|
|
}
|
|
}
|
|
|
|
$collection_1d = [];
|
|
foreach ($collection as $arr) {
|
|
foreach ($arr as $row) {
|
|
$collection_1d[] = $row;
|
|
}
|
|
}
|
|
SofaScorePlayerPerSeason::upsert(
|
|
$collection_1d,
|
|
[
|
|
'ss_player_id',
|
|
'ss_team_id',
|
|
'season',
|
|
],
|
|
[
|
|
'total_events',
|
|
'total_min',
|
|
'average_min',
|
|
'events_played_home',
|
|
'events_played_away',
|
|
'minutes_played_home',
|
|
'minutes_played_away',
|
|
'goals',
|
|
'fouls',
|
|
'offsides',
|
|
'shots_on_target',
|
|
'shots_off_target',
|
|
'cards',
|
|
]
|
|
);
|
|
}
|
|
}
|
|
|
|
private function array_template(array &$collection, int $ss_player_id, string $season)
|
|
{
|
|
$collection[$season][$ss_player_id]['ss_player_id'] = 0;
|
|
$collection[$season][$ss_player_id]['ss_team_id'] = 0;
|
|
$collection[$season][$ss_player_id]['season'] = '';
|
|
$collection[$season][$ss_player_id]['total_events'] = 0;
|
|
$collection[$season][$ss_player_id]['total_min'] = 0;
|
|
$collection[$season][$ss_player_id]['average_min'] = 0;
|
|
$collection[$season][$ss_player_id]['events_played_home'] = 0;
|
|
$collection[$season][$ss_player_id]['events_played_away'] = 0;
|
|
$collection[$season][$ss_player_id]['minutes_played_home'] = 0;
|
|
$collection[$season][$ss_player_id]['minutes_played_away'] = 0;
|
|
$collection[$season][$ss_player_id]['goals'] = 0;
|
|
$collection[$season][$ss_player_id]['fouls'] = 0;
|
|
$collection[$season][$ss_player_id]['offsides'] = 0;
|
|
$collection[$season][$ss_player_id]['shots_on_target'] = 0;
|
|
$collection[$season][$ss_player_id]['shots_off_target'] = 0;
|
|
$collection[$season][$ss_player_id]['cards'] = 0;
|
|
}
|
|
|
|
private function import08ManualStreamingWithPdo(string $filePath): void
|
|
{
|
|
// 100 7ms / 0MB
|
|
// 1K 78ms / 0MB
|
|
// 10K 328ms / 0MB
|
|
// 100K 2.9s / 0MB
|
|
// 1M 28s / 0MB
|
|
$data = [];
|
|
$handle = fopen($filePath, 'rb');
|
|
fgetcsv($handle); // skip header
|
|
$now = now()->format('Y-m-d H:i:s');
|
|
$pdo = DB::connection()->getPdo();
|
|
|
|
while (($row = fgetcsv($handle)) !== false) {
|
|
$data[] = [
|
|
'custom_id' => $row[0],
|
|
'name' => $row[1],
|
|
'email' => $row[2],
|
|
'company' => $row[3],
|
|
'city' => $row[4],
|
|
'country' => $row[5],
|
|
'birthday' => $row[6],
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
];
|
|
|
|
if (count($data) === 1000) {
|
|
// Build the SQL query for the chunk
|
|
$columns = array_keys($data[0]);
|
|
$placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
|
|
|
|
$sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
|
|
|
|
// Flatten the data array for the query
|
|
$values = [];
|
|
foreach ($data as $row) {
|
|
$values = array_merge($values, array_values($row));
|
|
}
|
|
|
|
$pdo->prepare($sql)->execute($values);
|
|
$data = [];
|
|
}
|
|
}
|
|
|
|
if (! empty($data)) {
|
|
$columns = array_keys($data[0]);
|
|
$placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
|
|
|
|
$sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
|
|
|
|
$values = [];
|
|
foreach ($data as $row) {
|
|
$values = array_merge($values, array_values($row));
|
|
}
|
|
|
|
$pdo->prepare($sql)->execute($values);
|
|
}
|
|
|
|
fclose($handle);
|
|
}
|
|
|
|
private function import09PDOPrepared(string $filePath): void
|
|
{
|
|
// Direct database connection with prepared statements
|
|
// 100 41ms / 0 MB
|
|
// 1K 237ms /
|
|
// 10K 2.21s
|
|
// 100K 25.27s
|
|
// 1M 4m43s
|
|
// 2M
|
|
$now = now()->format('Y-m-d H:i:s');
|
|
$handle = fopen($filePath, 'r');
|
|
fgets($handle); // skip header
|
|
|
|
try {
|
|
$pdo = DB::connection()->getPdo();
|
|
$stmt = $pdo->prepare('
|
|
INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
');
|
|
|
|
while (($row = fgetcsv($handle)) !== false) {
|
|
$stmt->execute([
|
|
$row[0],
|
|
$row[1],
|
|
$row[2],
|
|
$row[3],
|
|
$row[4],
|
|
$row[5],
|
|
$row[6],
|
|
$now,
|
|
$now,
|
|
]);
|
|
}
|
|
} finally {
|
|
fclose($handle);
|
|
}
|
|
}
|
|
|
|
private function import10PDOPreparedChunked(string $filePath): void
|
|
{
|
|
// Direct database connection with prepared statements
|
|
// 100 12ms / 0.15MB
|
|
// 1K 49ms / 0.74MB
|
|
// 10K 222ms / 0.74MB
|
|
// 100K 1.5s / 0.74MB
|
|
// 1M 15.3s / 0.74MB
|
|
// 2M 24s / 0.74MN
|
|
$now = now()->format('Y-m-d H:i:s');
|
|
$handle = fopen($filePath, 'r');
|
|
fgetcsv($handle); // skip header
|
|
$chunkSize = 500;
|
|
$chunks = [];
|
|
|
|
try {
|
|
$stmt = $this->prepareChunkedStatement($chunkSize);
|
|
|
|
while (($row = fgetcsv($handle)) !== false) {
|
|
$chunks = array_merge($chunks, [
|
|
$row[0], $row[1], $row[2], $row[3], $row[4],
|
|
$row[5], $row[6], $now, $now,
|
|
]);
|
|
|
|
if (count($chunks) === $chunkSize * 9) { // 9 columns
|
|
$stmt->execute($chunks);
|
|
$chunks = [];
|
|
}
|
|
}
|
|
|
|
// Handle remaining records
|
|
if (! empty($chunks)) {
|
|
$remainingRows = count($chunks) / 9;
|
|
$stmt = $this->prepareChunkedStatement($remainingRows);
|
|
$stmt->execute($chunks);
|
|
}
|
|
} finally {
|
|
fclose($handle);
|
|
}
|
|
}
|
|
|
|
private function import11Concurrent(string $filePath): void
|
|
{
|
|
// 100 168ms
|
|
// 1K 172ms
|
|
// 10K 234ms
|
|
// 100K 595ms
|
|
// 1M 4.36s
|
|
// 2M 8.8s
|
|
$now = now()->format('Y-m-d H:i:s');
|
|
$numberOfProcesses = 10;
|
|
|
|
$tasks = [];
|
|
for ($i = 0; $i < $numberOfProcesses; $i++) {
|
|
$tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
|
|
DB::reconnect();
|
|
|
|
$handle = fopen($filePath, 'r');
|
|
fgets($handle); // Skip header
|
|
$currentLine = 0;
|
|
$customers = [];
|
|
|
|
while (($line = fgets($handle)) !== false) {
|
|
// Each process takes every Nth line
|
|
if ($currentLine++ % $numberOfProcesses !== $i) {
|
|
continue;
|
|
}
|
|
|
|
$row = str_getcsv($line);
|
|
$customers[] = [
|
|
'custom_id' => $row[0],
|
|
'name' => $row[1],
|
|
'email' => $row[2],
|
|
'company' => $row[3],
|
|
'city' => $row[4],
|
|
'country' => $row[5],
|
|
'birthday' => $row[6],
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
];
|
|
|
|
if (count($customers) === 1000) {
|
|
DB::table('customers')->insert($customers);
|
|
$customers = [];
|
|
}
|
|
}
|
|
|
|
if (! empty($customers)) {
|
|
DB::table('customers')->insert($customers);
|
|
}
|
|
|
|
fclose($handle);
|
|
|
|
return true;
|
|
};
|
|
}
|
|
|
|
Concurrency::run($tasks);
|
|
|
|
}
|
|
|
|
private function import12LoadDataInfile(string $filePath): void
|
|
{
|
|
// MySQL specific, fastest approach
|
|
// 100 10ms / 0MB
|
|
// 1K 29ms / 0MB
|
|
// 10K 115ms / 0MB
|
|
// 100K 567ms / 0MB
|
|
// 1M 5s / 0MB
|
|
// 2M 11s / 0MB
|
|
$pdo = DB::connection()->getPdo();
|
|
$pdo->setAttribute(PDO::MYSQL_ATTR_LOCAL_INFILE, true);
|
|
|
|
$filepath = str_replace('\\', '/', $filePath);
|
|
|
|
$query = <<<SQL
|
|
LOAD DATA LOCAL INFILE '$filepath'
|
|
INTO TABLE customers
|
|
FIELDS TERMINATED BY ','
|
|
ENCLOSED BY '"'
|
|
LINES TERMINATED BY '\n'
|
|
IGNORE 1 LINES
|
|
(@col1, @col2, @col3, @col4, @col5, @col6, @col7)
|
|
SET
|
|
custom_id = @col1,
|
|
name = @col2,
|
|
email = @col3,
|
|
company = @col4,
|
|
city = @col5,
|
|
country = @col6,
|
|
birthday = @col7,
|
|
created_at = NOW(),
|
|
updated_at = NOW()
|
|
SQL;
|
|
|
|
$pdo->exec($query);
|
|
}
|
|
|
|
private function prepareChunkedStatement($chunkSize): PDOStatement
|
|
{
|
|
$rowPlaceholders = '(?, ?, ?, ?, ?, ?, ?, ?, ?)';
|
|
$placeholders = implode(',', array_fill(0, $chunkSize, $rowPlaceholders));
|
|
|
|
return DB::connection()->getPdo()->prepare("
|
|
INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
|
|
VALUES {$placeholders}
|
|
");
|
|
}
|
|
}
|