added jobs back in for the fetching and processing the mining ledgers

those jobs were taking extremely long times to complete, and jobs are better suited for them.
This commit is contained in:
2021-03-15 19:38:06 +09:00
parent 4c281934ff
commit 0ee3699b44
6 changed files with 238 additions and 139 deletions
@@ -16,6 +16,9 @@ use App\Library\Esi\Esi;
use App\Library\Helpers\LookupHelper;
use App\Library\Moons\MoonCalc;
//Jobs
use App\Jobs\Commands\MiningTaxes\ProcessMiningTaxesLedgersJob;
//App Models
use App\Models\MiningTax\Observer;
use App\Models\MiningTax\Ledger;
@@ -48,7 +51,6 @@ class FetchMiningTaxesLedgersJob implements ShouldQueue
private $charId;
private $corpId;
private $observerId;
private $esi;
/**
* Create a new job instance.
@@ -64,17 +66,6 @@ class FetchMiningTaxesLedgersJob implements ShouldQueue
$this->charId = $charId;
$this->corpId = $corpId;
$this->observerId = $observerId;
//Setup the ESI stuff
$esiHelper = new Esi;
//Setup the private esi variables
if(!$esiHelper->haveEsiScope($this->charId, 'esi-industry.read_corporation_mining.v1')) {
Log::critical('Character: ' . $this->charId . ' did not have the correct esi scope in FetchMiningTaxesLedgersJob.');
return null;
}
$refreshToken = $esiHelper->GetRefreshToken($this->charId);
$this->esi = $esiHelper->SetupEsiAuthentication($refreshToken);
}
/**
@@ -88,55 +79,75 @@ class FetchMiningTaxesLedgersJob implements ShouldQueue
$lookup = new LookupHelper;
$mHelper = new MoonCalc;
$esiHelper = new Esi;
$pageFailed = false;
//Get the ledger from ESI
try {
$response = $this->esi->invoke('get', '/corporation/{corporation_id}/mining/observers/{observer_id}/', [
'corporation_id' => $this->corpId,
'observer_id' => $this->observerId,
]);
} catch(RequestFailedException $e) {
Log::warning('Failed to get the mining ledger in FetchMiningTaxesLedgersJob for observer id: ' . $this->observerId);
//Check for the correct scope
if(!$esiHelper->haveEsiScope($this->charId, 'esi-industry.read_corporation_mining.v1')) {
Log::critical('Character: ' . $this->charId . ' did not have the correct esi scope in FetchMiningTaxesLedgersJob.');
return null;
}
$ledgers = json_decode($response);
//Get the esi token in order to pull data from esi
$refreshToken = $esiHelper->GetRefreshToken($this->charId);
$esi = $esiHelper->SetupEsiAuthentication($refreshToken);
//Sort through the array, and create the variables needed for database entries
foreach($ledgers as $ledger) {
//Get some basic information we need to work with
$charName = $lookup->CharacterIdToName($ledger->character_id);
//Get the type name from the ledger ore stuff
$typeName = $lookup->ItemIdToName($ledger->type_id);
//Decode the date and store it.
//$updated = $esiHelper->DecodeDate($ledger->last_updated);
//Set the current page
$currentPage = 1;
$totalPages = 1;
$price = $mHelper->CalculateOrePrice($ledger->type_id);
$amount = $price * $ledger->quantity;
//Setup a do-while loop to sort through the ledgers by pages
do {
/**
* During the course of the operation, we want to ensure our token stays valid.
* If the token, expires, then we want to refresh the token through the esi helper
* library functionality.
*/
if($esiHelper->TokenExpired($refreshToken)) {
$refreshToken = $esiHelper->GetRefreshToken($charId);
$esi = $esiHelper->SetupEsiAuthentication($refreshToken);
}
//Insert or update the entry in the database
$item = Ledger::updateOrCreate([
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $this->observerId,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'price' => $amount,
], [
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $this->observerId,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'price' => $amount,
]);
}
/**
* Attempt to get the data from the esi api. If it fails, we skip the page
*/
try {
$response = $esi->page($currentPage)
->invoke('get', '/corporation/{corporation_id}/mining/observers/{observer_id}/', [
'corporation_id' => $config['corporation'],
'observer_id' => $obs->observer_id,
]);
} catch(RequestFailedException $e) {
Log::warning('Failed to get the mining ledger in FetchMiningTaxesLedgersCommand for observer id: ' . $this->observerId);
$pageFailed = true;
}
//Clean up old data
Ledger::where(['updated_at', '<', Carbon::now()->subDays(120)])->delete();
/**
* If the current page is the first one and the page didn't fail, then update the total pages.
* If the first page failed, just return as we aren't going to be able to get the total amount of data needed.
*/
if($currentPage == 1 && $pageFailed == false) {
$totalPages = $response->pages;
} else if($currentPage == 1 && $pageFailed == true) {
return null;
}
if($pageFailed == true) {
//If the page failed, then reset the variable, and skip the current iteration
//of creating the jobs.
$pageFailed = false;
} else {
//Decode the json response from the ledgers
$ledgers = json_decode($response->raw);
//Dispatch jobs to process each of the mining ledger entries
foreach($ledgers as $ledger) {
ProcessMiningTaxesLedgersJob::dispatch($ledger)->onQueue('miningtaxes');
}
}
//Increment the current pages
$currentPage++;
} while($currentPage <= $totalPages);
}
}
@@ -0,0 +1,98 @@
<?php
namespace App\Jobs\Commands\MiningTaxes;
//Internal Library
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
//App Library
use App\Library\Helpers\LookupHelper;
use App\Library\Moons\MoonCalc;
//Models
use App\Models\MiningTax\Ledger;
use App\Models\Moon\MineralPrice;
use App\Models\Moon\ItemComposition;
class ProcessMiningTaxesLedgersJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Timeout in seconds
*
* @var int
*/
public $timeout = 3600;
/**
* Number of job retries
*
* @var int
*/
public $tries = 3;
/**
* Job Variables
*/
private $ledger;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($ledger)
{
//Set the connection for the job
$this->connection = 'redis';
//Import variables from the calling function
$this->ledger = $ledger;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$lookup = new LookupHelper;
$mHelper = new MoonCalc;
//Get some of the basic information we need to work with
$charName = $lookup->CharacterIdToName($this->ledger->character_id);
//Get the type name from the ledger ore
$typeName = $lookup->ItemIdToName($this->ledger->type_id);
//Get the price from the helper function
$price = $mHelper->CalculateOrePrice($this->ledger->type_id);
//Calculate the total price based on the amount
$amount = $price * $this->ledger->quantity;
//Insert or update the entry in the database
$item = Ledger::updateOrCreate([
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $obs->observer_id,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'amount' => $amount,
], [
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $obs->observer_id,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'amount' => $amount,
]);
}
}