salam/nuhodds_fi4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip import

Browse Source
This commit is contained in:
salam committed 2026-04-21 11:25:40 +02:00
1 parent 048989fed8
commit 4fa4e1a611
11 files changed
+48902 -141
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.phpstorm.meta.php
+12213
View File
File diff suppressed because it is too large. Load diff
_ide_helper.php
+34417
View File
File diff suppressed because it is too large. Load diff
_ide_helper_models.php
+2123
View File
File diff suppressed because it is too large. Load diff
app/Filament/Clusters/StaticData/Resources/StatistikoImports/Schemas/StatistikoImportForm.php
+1 -1
View File
@@ -40,7 +40,7 @@ class StatistikoImportForm
->acceptedFileTypes([
'application/vnd.ms-excel',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'text/csv', // Optional: include if you accept CSV too
// 'text/csv', // Optional: include if you accept CSV too
]),
])->columns(1);
}
app/Filament/Clusters/StaticData/Resources/StatistikoImports/Tables/StatistikoImportsTable.php
+3 -14
View File
@@ -60,25 +60,14 @@ class StatistikoImportsTable
->recordActions([
ActionGroup::make([
Action::make('Import')
->label('Import (fast)')
->label('Import to DB')
->icon('phosphor-database')
->action(function ($record) {
StatistikoImporter::run('import', $record);
})->successNotification(
->action(fn ($record) => StatistikoImporter::run($record))
->successNotification(
Notification::make()
->info()
->title('Import started')
),
Action::make('Update')
->label('Update or Import (slow)')
->icon('phosphor-database')
->action(function ($record) {
StatistikoImporter::run('update', $record);
})->successNotification(
Notification::make()
->info()
->title('Import & Update started')
),
DeleteAction::make()
->label('Delete file'),
]),
app/Filament/Clusters/StaticData/Resources/StatistikoPlayers/Tables/StatistikoPlayersTable.php
+10 -10
View File
@@ -19,18 +19,18 @@ class StatistikoPlayersTable
->paginationPageOptions([8, 16, 24, 36])
->columns([
Stack::make([
TextColumn::make('id_giocatore')->prefix('Id: ')->searchable()->color('success')->icon('phosphor-copy')->copyable(),
TextColumn::make('giocatore')->searchable()->prefix('Nome: ')->color('primary')->icon('phosphor-copy')->copyable(),
TextColumn::make('nome_completo')->color('gray')->searchable(),
TextColumn::make('data_di_nascita')->date()->prefix('Nascita: '),
TextColumn::make('altezza')->prefix('Altezza: ')->suffix(' cm'),
TextColumn::make('nazionalita')->prefix('Nazionalita: '),
TextColumn::make('posizione')->prefix('Posizione: '),
TextColumn::make('player_id')->prefix('Id: ')->searchable()->color('success')->icon('phosphor-copy')->copyable(),
TextColumn::make('name')->searchable()->prefix('Name: ')->color('primary')->icon('phosphor-copy')->copyable(),
TextColumn::make('full_name')->color('gray')->searchable(),
TextColumn::make('birthdate')->date()->prefix('Birthdate: '),
TextColumn::make('height')->prefix('Height: ')->suffix(' cm'),
TextColumn::make('nationality')->prefix('Nationality: '),
TextColumn::make('position')->prefix('Position: '),
TextColumn::make('sport_radar_id')->searchable()->prefix('Sport Radar Id: ')->icon('phosphor-copy')->copyable(),
TextColumn::make('id_squadra_tm')->prefix('Id Squadra TM: '),
TextColumn::make('rigorista')->prefix('Rigorista: '),
TextColumn::make('id_squadra_tm')->prefix('Team Id TM: '),
TextColumn::make('rigorist')->prefix('Rigorist: '),
TextColumn::make('national_team_id')->prefix('National Team Id: '),
TextColumn::make('data_aggiornamento')->date()->prefix('Aggiornamento: '),
TextColumn::make('last_update')->date()->prefix('Last Update: '),
TextColumn::make('created_at')->date()->prefix('Created: '),
TextColumn::make('updated_at')->date()->prefix('Updated: '),
// TextColumn::make('league_code')->prefix('League: '),
app/Filament/Imports/StatistikoImporter.php
+89 -100
View File
@@ -4,11 +4,10 @@ namespace App\Filament\Imports;
use App\Models\StatistikoPlayer;
use Filament\Notifications\Notification;
use Illuminate\Support\Facades\Concurrency;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\HtmlString;
use Illuminate\Support\LazyCollection;
use Illuminate\Support\Str;
use Spatie\SimpleExcel\SimpleExcelReader;
class StatistikoImporter
@@ -21,25 +20,29 @@ class StatistikoImporter
public static int $startQueries;
public static function run($mode, $record): void
public static function run($record): void
{
error_reporting(E_ALL);
ini_set('display_errors', 1);
ini_set('max_execution_time', -1);
$table = $record['data_type'];
$filePath = storage_path().'/app/private/'.$record['file_name'];
$filePath = $record['file_name'];
self::startBenchmark($table);
try {
self::handleImport($filePath, $table, $mode);
} catch (\Exception $e) {
dd(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value());
switch ($record['data_type']) {
case 'player':
self::importPlayer($filePath);
break;
case 'events':
self::importEvents($filePath);
break;
case 'events_player':
break;
}
self::endBenchmark();
self::endBenchmark($table);
}
protected static function startBenchmark(string $table): void
@@ -50,7 +53,7 @@ class StatistikoImporter
DB::connection('mongodb')->enableQueryLog();
}
protected static function endBenchmark(string $table = 'customers'): void
protected static function endBenchmark(string $table): void
{
$executionTime = microtime(true) - self::$benchmarkStartTime;
$memoryUsage = round((memory_get_usage() - self::$benchmarkStartMemory) / 1024 / 1024, 2);
@@ -73,6 +76,7 @@ class StatistikoImporter
$str .= '<br/>ROWS: '.number_format($rowDiff);
$recipient = auth()->user();
Notification::make()
->success()
->title('Import compleated')
@@ -80,110 +84,95 @@ class StatistikoImporter
->sendToDatabase($recipient);
}
private static function handleImport(string $filePath, $table, $mode): void
private static function importPlayer(string $filePath): void
{
$now = now()->format('Y-m-d H:i:s');
$chunkSize = 1000;
SimpleExcelReader::create($filePath)
SimpleExcelReader::create(storage_path('app/private/'.$filePath))
->getRows()
->chunk($chunkSize)
->each(function ($rowProperties) {
dd($rowProperties);
// processing rows
->chunk(1000)
->each(function (LazyCollection $chunk) use ($now) {
$data = [];
foreach ($chunk as $row) {
$row = array_values($row);
// check if name exists
if (is_string(preg_replace("'\W'", '', $row[1]))) {
$data[] = [
'player_id' => $row[0],
'name' => $row[1],
'birthdate' => Carbon::parse($row[2])->toDateString(),
'height' => $row[3],
'nationality' => $row[4],
'position' => $row[5],
'sport_radar_id' => $row[6],
'id_squadra_tm' => $row[7],
'full_name' => $row[8],
'no_specialities' => $row[9],
'rigorist' => $row[10],
'national_team_id' => $row[11],
'last_update' => Carbon::parse($row[12])->toDateString(),
'league_code' => $row[13],
'team_code' => $row[14],
'team_name' => $row[15],
'created_at' => $now,
'updated_at' => $now,
];
}
}
StatistikoPlayer::upsert($data, 'player_id');
});
// LazyCollection::make(function () use ($filePath) {
// $handle = fopen($filePath, 'r');
// fgets($handle); // skip header
// while (($line = fgets($handle)) !== false) {
// yield str_getcsv($line);
// }
// fclose($handle);
// })->each(function ($row) {
// dd($row);
// });
// ->map(fn ($row) => [
// 'custom_id' => $row[0],
// 'name' => $row[1],
// 'email' => $row[2],
// 'company' => $row[3],
// 'city' => $row[4],
// 'country' => $row[5],
// 'birthday' => $row[6],
// 'created_at' => $now,
// 'updated_at' => $now,
// ])
// ->chunk($chunkSize)
// ->each(fn ($chunk) => $result = StatistikoPlayer::upsert(
// [
// ['title' => 'Inspector Maigret', 'recommended' => false, 'runtime' => 128],
// ['title' => 'Petit Maman', 'recommended' => true, 'runtime' => 72],
// ],
// 'title',
// 'recommended',
// ));
}
private static function import11Concurrent(string $filePath): void
private static function importEvents(string $filePath): void
{
// 100 168ms
// 1K 172ms
// 10K 234ms
// 100K 595ms
// 1M 4.36s
// 2M 8.8s
$now = now()->format('Y-m-d H:i:s');
$numberOfProcesses = 10;
$tasks = [];
for ($i = 0; $i < $numberOfProcesses; $i++) {
$tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
DB::reconnect();
SimpleExcelReader::create(storage_path('app/private/'.$filePath))
->getRows()
->chunk(1000)
->each(function (LazyCollection $chunk) use ($now) {
$handle = fopen($filePath, 'r');
fgets($handle); // Skip header
$currentLine = 0;
$customers = [];
$data = [];
while (($line = fgets($handle)) !== false) {
// Each process takes every Nth line
if ($currentLine++ % $numberOfProcesses !== $i) {
continue;
}
foreach ($chunk as $row) {
$row = str_getcsv($line);
$customers[] = [
'custom_id' => $row[0],
'name' => $row[1],
'email' => $row[2],
'company' => $row[3],
'city' => $row[4],
'country' => $row[5],
'birthday' => $row[6],
$row = array_values($row);
dd($row);
$data[] = [
'sport_id' => 1,
'league_id' => $row[1],
'home_id' => $row[2],
'away_id' => $row[3],
'starttime' => Carbon::parse($row[4])->toDateTimeString(),
'season' => $row[5],
'gls_h' => $row[6],
'gls_a' => $row[7],
'crn_h' => $row[8],
'fls_h' => $row[9],
'ofs_h' => $row[10],
'snt_h' => $row[11],
'tsh_h' => $row[12],
'crd_h' => $row[13],
'crn_a' => $row[14],
'fls_a' => $row[15],
'ofs_a' => $row[16],
'snt_a' => $row[17],
'tsh_a' => $row[18],
'crd_a' => $row[19],
'expgoalshome_value' => $row[20],
'expgoalsaway_value' => $row[21],
'exp_handicap' => $row[22],
'exp_total' => $row[23],
'created_at' => $now,
'updated_at' => $now,
];
if (count($customers) === 1000) {
DB::table('customers')->insert($customers);
$customers = [];
}
}
if (! empty($customers)) {
DB::table('customers')->insert($customers);
}
fclose($handle);
return true;
};
}
Concurrency::run($tasks);
StatistikoPlayer::upsert($data, 'player_id');
});
}
}
app/Models/StatistikoImport.php
+1 -1
View File
@@ -12,8 +12,8 @@ class StatistikoImport extends Model
'data_type',
'file_name',
'file_name_original',
'file_path',
'run_import',
'has_run',
];
public function sport()
composer.lock
Generated
+12 -12
View File
@@ -9795,16 +9795,16 @@
"packages-dev": [
{
"name": "barryvdh/laravel-debugbar",
"version": "v4.2.6",
"version": "v4.2.7",
"source": {
"type": "git",
"url": "https://github.com/fruitcake/laravel-debugbar.git",
"reference": "68c3de788feb7047bea547b51253e71a54e7936a"
"reference": "e9863784b8ee42f7a4393bea90a0750f632df779"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fruitcake/laravel-debugbar/zipball/68c3de788feb7047bea547b51253e71a54e7936a",
"reference": "68c3de788feb7047bea547b51253e71a54e7936a",
"url": "https://api.github.com/repos/fruitcake/laravel-debugbar/zipball/e9863784b8ee42f7a4393bea90a0750f632df779",
"reference": "e9863784b8ee42f7a4393bea90a0750f632df779",
"shasum": ""
},
"require": {
@@ -9878,7 +9878,7 @@
],
"support": {
"issues": "https://github.com/fruitcake/laravel-debugbar/issues",
"source": "https://github.com/fruitcake/laravel-debugbar/tree/v4.2.6"
"source": "https://github.com/fruitcake/laravel-debugbar/tree/v4.2.7"
},
"funding": [
{
@@ -9890,7 +9890,7 @@
"type": "github"
}
],
"time": "2026-04-10T07:24:18+00:00"
"time": "2026-04-18T17:01:20+00:00"
},
{
"name": "brianium/paratest",
@@ -11347,16 +11347,16 @@
},
{
"name": "php-debugbar/php-debugbar",
"version": "v3.7.4",
"version": "v3.7.5",
"source": {
"type": "git",
"url": "https://github.com/php-debugbar/php-debugbar.git",
"reference": "af82a54530c56919ce3c6fba1942a247ac9f6777"
"reference": "dbf77f48fa6e6b57ed57ae67aa047b2535697788"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-debugbar/php-debugbar/zipball/af82a54530c56919ce3c6fba1942a247ac9f6777",
"reference": "af82a54530c56919ce3c6fba1942a247ac9f6777",
"url": "https://api.github.com/repos/php-debugbar/php-debugbar/zipball/dbf77f48fa6e6b57ed57ae67aa047b2535697788",
"reference": "dbf77f48fa6e6b57ed57ae67aa047b2535697788",
"shasum": ""
},
"require": {
@@ -11433,7 +11433,7 @@
],
"support": {
"issues": "https://github.com/php-debugbar/php-debugbar/issues",
"source": "https://github.com/php-debugbar/php-debugbar/tree/v3.7.4"
"source": "https://github.com/php-debugbar/php-debugbar/tree/v3.7.5"
},
"funding": [
{
@@ -11445,7 +11445,7 @@
"type": "github"
}
],
"time": "2026-04-09T11:28:41+00:00"
"time": "2026-04-15T11:58:43+00:00"
},
{
"name": "php-debugbar/symfony-bridge",
database/migrations/2026_04_20_145255_mongo_db_statistiko_event.php
+30
View File
@@ -0,0 +1,30 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Support\Facades\Schema;
use MongoDB\Laravel\Schema\Blueprint;
return new class extends Migration
{
protected $connection = 'mongodb';
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('events', function (Blueprint $collection) {
$collection->index('league_id');
$collection->index('home_id');
$collection->index('away_id');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('events');
}
};
package-lock.json
Generated
+3 -3
View File
@@ -1220,9 +1220,9 @@
"license": "MIT"
},
"node_modules/axios": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz",
"integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==",
"version": "1.15.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.15.1.tgz",
"integrity": "sha512-WOG+Jj8ZOvR0a3rAn+Tuf1UQJRxw5venr6DgdbJzngJE3qG7X0kL83CZGpdHMxEm+ZK3seAbvFsw4FfOfP9vxg==",
"dev": true,
"license": "MIT",
"dependencies": {