salam/nuhodds_fi4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
salam committed 2026-09-12 11:46:36 +02:00
1 parent 4447a2d97a
commit abf9a913e0
3 files changed
+218 -215
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Models/SofaScorePlayerPerSeason.php
+1
View File
@@ -25,6 +25,7 @@ class SofaScorePlayerPerSeason extends Model
'offsides', 'offsides',
'shots_on_target', 'shots_on_target',
'shots_off_target', 'shots_off_target',
'total_shots',
]; ];
public function details() public function details()
app/Traits/UpdateHelper.php
+216 -215
View File
@@ -6,11 +6,8 @@ use App\Models\SofaScorePlayerPerSeason;
use App\Models\StatistikoDetail; use App\Models\StatistikoDetail;
use App\Models\StatistikoPlayer; use App\Models\StatistikoPlayer;
use App\Utilities\Notice; use App\Utilities\Notice;
use Illuminate\Support\Facades\Concurrency;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Str; use Illuminate\Support\Str;
use PDO;
use PDOStatement;
use function Laravel\Prompts\select; use function Laravel\Prompts\select;
@@ -172,6 +169,7 @@ trait UpdateHelper
$collection[$season][$ss_player_id]['offsides'] += $val->offsides; $collection[$season][$ss_player_id]['offsides'] += $val->offsides;
$collection[$season][$ss_player_id]['shots_on_target'] += $val->shots_on_target; $collection[$season][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
$collection[$season][$ss_player_id]['shots_off_target'] += $val->shots_off_target; $collection[$season][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
$collection[$season][$ss_player_id]['total_shots'] += $val->shots_off_target + $val->shots_off_target + $val->shots_blocked;
$collection['total'][$ss_player_id]['ss_player_id'] = $ss_player_id; $collection['total'][$ss_player_id]['ss_player_id'] = $ss_player_id;
$collection['total'][$ss_player_id]['ss_team_id'] = $ss_team_id; $collection['total'][$ss_player_id]['ss_team_id'] = $ss_team_id;
@@ -185,6 +183,7 @@ trait UpdateHelper
$collection['total'][$ss_player_id]['offsides'] += $val->offsides; $collection['total'][$ss_player_id]['offsides'] += $val->offsides;
$collection['total'][$ss_player_id]['shots_on_target'] += $val->shots_on_target; $collection['total'][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
$collection['total'][$ss_player_id]['shots_off_target'] += $val->shots_off_target; $collection['total'][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
$collection['total'][$ss_player_id]['total_shots'] += $val->shots_off_target + $val->shots_off_target + $val->shots_blocked;
} }
} }
} }
@@ -216,6 +215,7 @@ trait UpdateHelper
'offsides', 'offsides',
'shots_on_target', 'shots_on_target',
'shots_off_target', 'shots_off_target',
'total_shots',
'cards', 'cards',
] ]
); );
@@ -239,254 +239,255 @@ trait UpdateHelper
$collection[$season][$ss_player_id]['offsides'] = 0; $collection[$season][$ss_player_id]['offsides'] = 0;
$collection[$season][$ss_player_id]['shots_on_target'] = 0; $collection[$season][$ss_player_id]['shots_on_target'] = 0;
$collection[$season][$ss_player_id]['shots_off_target'] = 0; $collection[$season][$ss_player_id]['shots_off_target'] = 0;
$collection[$season][$ss_player_id]['total_shots'] = 0;
$collection[$season][$ss_player_id]['cards'] = 0; $collection[$season][$ss_player_id]['cards'] = 0;
} }
private function import08ManualStreamingWithPdo(string $filePath): void // private function import08ManualStreamingWithPdo(string $filePath): void
{ // {
// 100 7ms / 0MB // // 100 7ms / 0MB
// 1K 78ms / 0MB // // 1K 78ms / 0MB
// 10K 328ms / 0MB // // 10K 328ms / 0MB
// 100K 2.9s / 0MB // // 100K 2.9s / 0MB
// 1M 28s / 0MB // // 1M 28s / 0MB
$data = []; // $data = [];
$handle = fopen($filePath, 'rb'); // $handle = fopen($filePath, 'rb');
fgetcsv($handle); // skip header // fgetcsv($handle); // skip header
$now = now()->format('Y-m-d H:i:s'); // $now = now()->format('Y-m-d H:i:s');
$pdo = DB::connection()->getPdo(); // $pdo = DB::connection()->getPdo();
while (($row = fgetcsv($handle)) !== false) { // while (($row = fgetcsv($handle)) !== false) {
$data[] = [ // $data[] = [
'custom_id' => $row[0], // 'custom_id' => $row[0],
'name' => $row[1], // 'name' => $row[1],
'email' => $row[2], // 'email' => $row[2],
'company' => $row[3], // 'company' => $row[3],
'city' => $row[4], // 'city' => $row[4],
'country' => $row[5], // 'country' => $row[5],
'birthday' => $row[6], // 'birthday' => $row[6],
'created_at' => $now, // 'created_at' => $now,
'updated_at' => $now, // 'updated_at' => $now,
]; // ];
if (count($data) === 1000) { // if (count($data) === 1000) {
// Build the SQL query for the chunk // // Build the SQL query for the chunk
$columns = array_keys($data[0]); // $columns = array_keys($data[0]);
$placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ','); // $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
$sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders; // $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
// Flatten the data array for the query // // Flatten the data array for the query
$values = []; // $values = [];
foreach ($data as $row) { // foreach ($data as $row) {
$values = array_merge($values, array_values($row)); // $values = array_merge($values, array_values($row));
} // }
$pdo->prepare($sql)->execute($values); // $pdo->prepare($sql)->execute($values);
$data = []; // $data = [];
} // }
} // }
if (! empty($data)) { // if (! empty($data)) {
$columns = array_keys($data[0]); // $columns = array_keys($data[0]);
$placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ','); // $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
$sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders; // $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
$values = []; // $values = [];
foreach ($data as $row) { // foreach ($data as $row) {
$values = array_merge($values, array_values($row)); // $values = array_merge($values, array_values($row));
} // }
$pdo->prepare($sql)->execute($values); // $pdo->prepare($sql)->execute($values);
} // }
fclose($handle); // fclose($handle);
} // }
private function import09PDOPrepared(string $filePath): void // private function import09PDOPrepared(string $filePath): void
{ // {
// Direct database connection with prepared statements // // Direct database connection with prepared statements
// 100 41ms / 0 MB // // 100 41ms / 0 MB
// 1K 237ms / // // 1K 237ms /
// 10K 2.21s // // 10K 2.21s
// 100K 25.27s // // 100K 25.27s
// 1M 4m43s // // 1M 4m43s
// 2M // // 2M
$now = now()->format('Y-m-d H:i:s'); // $now = now()->format('Y-m-d H:i:s');
$handle = fopen($filePath, 'r'); // $handle = fopen($filePath, 'r');
fgets($handle); // skip header // fgets($handle); // skip header
try { // try {
$pdo = DB::connection()->getPdo(); // $pdo = DB::connection()->getPdo();
$stmt = $pdo->prepare(' // $stmt = $pdo->prepare('
INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at) // INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) // VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
'); // ');
while (($row = fgetcsv($handle)) !== false) { // while (($row = fgetcsv($handle)) !== false) {
$stmt->execute([ // $stmt->execute([
$row[0], // $row[0],
$row[1], // $row[1],
$row[2], // $row[2],
$row[3], // $row[3],
$row[4], // $row[4],
$row[5], // $row[5],
$row[6], // $row[6],
$now, // $now,
$now, // $now,
]); // ]);
} // }
} finally { // } finally {
fclose($handle); // fclose($handle);
} // }
} // }
private function import10PDOPreparedChunked(string $filePath): void // private function import10PDOPreparedChunked(string $filePath): void
{ // {
// Direct database connection with prepared statements // // Direct database connection with prepared statements
// 100 12ms / 0.15MB // // 100 12ms / 0.15MB
// 1K 49ms / 0.74MB // // 1K 49ms / 0.74MB
// 10K 222ms / 0.74MB // // 10K 222ms / 0.74MB
// 100K 1.5s / 0.74MB // // 100K 1.5s / 0.74MB
// 1M 15.3s / 0.74MB // // 1M 15.3s / 0.74MB
// 2M 24s / 0.74MN // // 2M 24s / 0.74MN
$now = now()->format('Y-m-d H:i:s'); // $now = now()->format('Y-m-d H:i:s');
$handle = fopen($filePath, 'r'); // $handle = fopen($filePath, 'r');
fgetcsv($handle); // skip header // fgetcsv($handle); // skip header
$chunkSize = 500; // $chunkSize = 500;
$chunks = []; // $chunks = [];
try { // try {
$stmt = $this->prepareChunkedStatement($chunkSize); // $stmt = $this->prepareChunkedStatement($chunkSize);
while (($row = fgetcsv($handle)) !== false) { // while (($row = fgetcsv($handle)) !== false) {
$chunks = array_merge($chunks, [ // $chunks = array_merge($chunks, [
$row[0], $row[1], $row[2], $row[3], $row[4], // $row[0], $row[1], $row[2], $row[3], $row[4],
$row[5], $row[6], $now, $now, // $row[5], $row[6], $now, $now,
]); // ]);
if (count($chunks) === $chunkSize * 9) { // 9 columns // if (count($chunks) === $chunkSize * 9) { // 9 columns
$stmt->execute($chunks); // $stmt->execute($chunks);
$chunks = []; // $chunks = [];
} // }
} // }
// Handle remaining records // // Handle remaining records
if (! empty($chunks)) { // if (! empty($chunks)) {
$remainingRows = count($chunks) / 9; // $remainingRows = count($chunks) / 9;
$stmt = $this->prepareChunkedStatement($remainingRows); // $stmt = $this->prepareChunkedStatement($remainingRows);
$stmt->execute($chunks); // $stmt->execute($chunks);
} // }
} finally { // } finally {
fclose($handle); // fclose($handle);
} // }
} // }
private function import11Concurrent(string $filePath): void // private function import11Concurrent(string $filePath): void
{ // {
// 100 168ms // // 100 168ms
// 1K 172ms // // 1K 172ms
// 10K 234ms // // 10K 234ms
// 100K 595ms // // 100K 595ms
// 1M 4.36s // // 1M 4.36s
// 2M 8.8s // // 2M 8.8s
$now = now()->format('Y-m-d H:i:s'); // $now = now()->format('Y-m-d H:i:s');
$numberOfProcesses = 10; // $numberOfProcesses = 10;
$tasks = []; // $tasks = [];
for ($i = 0; $i < $numberOfProcesses; $i++) { // for ($i = 0; $i < $numberOfProcesses; $i++) {
$tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) { // $tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
DB::reconnect(); // DB::reconnect();
$handle = fopen($filePath, 'r'); // $handle = fopen($filePath, 'r');
fgets($handle); // Skip header // fgets($handle); // Skip header
$currentLine = 0; // $currentLine = 0;
$customers = []; // $customers = [];
while (($line = fgets($handle)) !== false) { // while (($line = fgets($handle)) !== false) {
// Each process takes every Nth line // // Each process takes every Nth line
if ($currentLine++ % $numberOfProcesses !== $i) { // if ($currentLine++ % $numberOfProcesses !== $i) {
continue; // continue;
} // }
$row = str_getcsv($line); // $row = str_getcsv($line);
$customers[] = [ // $customers[] = [
'custom_id' => $row[0], // 'custom_id' => $row[0],
'name' => $row[1], // 'name' => $row[1],
'email' => $row[2], // 'email' => $row[2],
'company' => $row[3], // 'company' => $row[3],
'city' => $row[4], // 'city' => $row[4],
'country' => $row[5], // 'country' => $row[5],
'birthday' => $row[6], // 'birthday' => $row[6],
'created_at' => $now, // 'created_at' => $now,
'updated_at' => $now, // 'updated_at' => $now,
]; // ];
if (count($customers) === 1000) { // if (count($customers) === 1000) {
DB::table('customers')->insert($customers); // DB::table('customers')->insert($customers);
$customers = []; // $customers = [];
} // }
} // }
if (! empty($customers)) { // if (! empty($customers)) {
DB::table('customers')->insert($customers); // DB::table('customers')->insert($customers);
} // }
fclose($handle); // fclose($handle);
return true; // return true;
}; // };
} // }
Concurrency::run($tasks); // Concurrency::run($tasks);
} // }
private function import12LoadDataInfile(string $filePath): void // private function import12LoadDataInfile(string $filePath): void
{ // {
// MySQL specific, fastest approach // // MySQL specific, fastest approach
// 100 10ms / 0MB // // 100 10ms / 0MB
// 1K 29ms / 0MB // // 1K 29ms / 0MB
// 10K 115ms / 0MB // // 10K 115ms / 0MB
// 100K 567ms / 0MB // // 100K 567ms / 0MB
// 1M 5s / 0MB // // 1M 5s / 0MB
// 2M 11s / 0MB // // 2M 11s / 0MB
$pdo = DB::connection()->getPdo(); // $pdo = DB::connection()->getPdo();
$pdo->setAttribute(PDO::MYSQL_ATTR_LOCAL_INFILE, true); // $pdo->setAttribute(PDO::MYSQL_ATTR_LOCAL_INFILE, true);
$filepath = str_replace('\\', '/', $filePath); // $filepath = str_replace('\\', '/', $filePath);
$query = <<<SQL // $query = <<<SQL
LOAD DATA LOCAL INFILE '$filepath' // LOAD DATA LOCAL INFILE '$filepath'
INTO TABLE customers // INTO TABLE customers
FIELDS TERMINATED BY ',' // FIELDS TERMINATED BY ','
ENCLOSED BY '"' // ENCLOSED BY '"'
LINES TERMINATED BY '\n' // LINES TERMINATED BY '\n'
IGNORE 1 LINES // IGNORE 1 LINES
(@col1, @col2, @col3, @col4, @col5, @col6, @col7) // (@col1, @col2, @col3, @col4, @col5, @col6, @col7)
SET // SET
custom_id = @col1, // custom_id = @col1,
name = @col2, // name = @col2,
email = @col3, // email = @col3,
company = @col4, // company = @col4,
city = @col5, // city = @col5,
country = @col6, // country = @col6,
birthday = @col7, // birthday = @col7,
created_at = NOW(), // created_at = NOW(),
updated_at = NOW() // updated_at = NOW()
SQL; // SQL;
$pdo->exec($query); // $pdo->exec($query);
} // }
private function prepareChunkedStatement($chunkSize): PDOStatement // private function prepareChunkedStatement($chunkSize): PDOStatement
{ // {
$rowPlaceholders = '(?, ?, ?, ?, ?, ?, ?, ?, ?)'; // $rowPlaceholders = '(?, ?, ?, ?, ?, ?, ?, ?, ?)';
$placeholders = implode(',', array_fill(0, $chunkSize, $rowPlaceholders)); // $placeholders = implode(',', array_fill(0, $chunkSize, $rowPlaceholders));
return DB::connection()->getPdo()->prepare(" // return DB::connection()->getPdo()->prepare("
INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at) // INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
VALUES {$placeholders} // VALUES {$placeholders}
"); // ");
} // }
} }
database/migrations/2026_06_27_091506_create_sofa_score_player_per_seasons_table.php
+1
View File
@@ -29,6 +29,7 @@ return new class extends Migration
$table->integer('offsides')->default(0)->nullable(); $table->integer('offsides')->default(0)->nullable();
$table->integer('shots_on_target')->default(0)->nullable(); $table->integer('shots_on_target')->default(0)->nullable();
$table->integer('shots_off_target')->default(0)->nullable(); $table->integer('shots_off_target')->default(0)->nullable();
$table->integer('total_shots')->default(0)->nullable();
$table->unique(['ss_player_id', 'ss_team_id', 'season']); $table->unique(['ss_player_id', 'ss_team_id', 'season']);
$table->timestamps(); $table->timestamps();
}); });