Skip to content

Письменов Дмитрий Иванович / yourroomads

Go to a project
Commit afbd8523 by Vladislav
Options

#20434 Загрузка и синхронизация sitelinks

1 parent fceee760
Inline Side-by-side
Showing with 628 additions and 0 deletions
app/Console/Commands/DictionaryCampaignsSyncByCampaign.php
......@@ -156,6 +156,16 @@ class DictionaryCampaignsSyncByCampaign extends Command
//грузим расширения которых по какой то причне нет в целевых.
DB::insert("
INSERT INTO goal_sitelinks(sitelink_id, token_id, created_at, updated_at)
SELECT s.id, t.id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
FROM sitelinks s
INNER JOIN tokens t on t.type != '" . Tokens::MAIN . "'
LEFT JOIN goal_sitelinks gs on s.id = gs.sitelink_id and t.id = gs.token_id
WHERE gs.sitelink_id is null AND gs.deleted_at is null
");
//грузим наборы быстрых ссылок которых по какой то причне нет в целевых.
DB::insert("
INSERT INTO goal_ad_extensions(ad_extension_id, token_id, created_at, updated_at)
SELECT ae.id, t.id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
FROM ad_extensions ae
......
app/Console/Commands/SitelinksAdd.php 0 → 100644
<?php
namespace App\Console\Commands;
use App\Models\Pivots\GoalAdExtension;
use App\Models\Pivots\GoalSitelink;
use App\Models\Tokens;
use App\Service\API\API;
use App\Service\Requests\APIRequest;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Database\Eloquent\Relations\HasMany;
class SitelinksAdd extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'sitelinks:add';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Добавление не созданных наборов быстрых ссылок с целевого аккаунта';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
* @throws \Exception
*/
public function handle()
{
$tokens = Tokens::whereHas('goalSitelinksForNotExternalForNotReserveCreate.sitelink')
->where('type', '!=', Tokens::MAIN)
->get();
foreach ($tokens as $token) {
$token->load([
'goalSitelinksForNotExternalForNotReserveCreate' => function (HasMany $query) {
return $query->has('sitelink');
},
'goalSitelinksForNotExternalForNotReserveCreate.sitelink',
]);
$factory = APIRequest::getInstance(API::YANDEX);
$factory->setToken($token);
$goalSitelinks = $token->goalSitelinksForNotExternalForNotReserveCreate;
foreach (array_chunk($goalSitelinks->pluck('id')->toArray(), 1000) as $items) {
GoalSitelink::whereIn('id', $items)
->update([
'reserve_create_at' => Carbon::now(),
]);
}
$factory->getRequest('Sitelinks', 'add')
->call([
'goalSitelinks' => $goalSitelinks,
]);
}
return 0;
}
}
app/Console/Commands/SitelinksLoad.php 0 → 100644
<?php
namespace App\Console\Commands;
use App\Models\Campaigns;
use App\Models\Tokens;
use App\Service\API\API;
use App\Service\Requests\APIRequest;
use Illuminate\Console\Command;
class SitelinksLoad extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'sitelinks:load';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Загрузка наборы быстрых ссылок';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
* @throws \Exception
*/
public function handle()
{
$token = Tokens::where('type', Tokens::MAIN)->first();
if (!$token) {
throw new \Exception('Не найден токен блин');
}
$factory = APIRequest::getInstance(API::YANDEX);
$factory->setToken($token);
$factory->getRequest('Sitelinks', 'get')
->call();
return 0;
}
}
app/Console/Kernel.php
......@@ -25,6 +25,8 @@ use App\Console\Commands\KeywordsAdd;
use App\Console\Commands\KeywordsDelete;
use App\Console\Commands\KeywordsUpdate;
use App\Console\Commands\RefreshLimits;
use App\Console\Commands\SitelinksAdd;
use App\Console\Commands\SitelinksLoad;
use App\Console\Commands\VCardsAdd;
use App\Console\Commands\VCardsLoad;
use Illuminate\Console\Scheduling\Schedule;
......@@ -71,6 +73,9 @@ class Kernel extends ConsoleKernel
$schedule->command(AdExtensionsLoad::class)->hourlyAt(25);
$schedule->command(AdExtensionsAdd::class)->hourlyAt(30);
$schedule->command(SitelinksLoad::class)->hourlyAt(25);
$schedule->command(SitelinksAdd::class)->hourlyAt(30);
$schedule->command(AdGroupsAdd::class)->hourlyAt(30);
$schedule->command(AdGroupsUpdate::class)->hourlyAt(30);
......
app/Models/Pivots/GoalSitelink.php 0 → 100644
<?php
namespace App\Models\Pivots;
use App\Models\Sitelink;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Relations\Pivot;
use Illuminate\Database\Eloquent\SoftDeletes;
class GoalSitelink extends Pivot
{
use SoftDeletes;
protected $table = 'goal_sitelinks';
protected $fillable = [
'external_id',
'sitelink_id',
'token_id',
'external_upload_at',
'external_updated_at',
'updated_need',
'reserve_create_at',
'reserve_update_at',
];
protected $casts = [
'external_upload_at' => 'datetime',
'external_updated_at' => 'datetime',
'updated_need' => 'datetime',
'reserve_create_at' => 'datetime',
'reserve_update_at' => 'datetime',
];
public $incrementing = true;
static public function getWithPivot()
{
return [
'id',
'external_id',
'sitelink_id',
'token_id',
'external_upload_at',
'external_updated_at',
'updated_need',
'reserve_create_at',
'reserve_update_at',
];
}
/**
* @param Builder $query
* @return Builder
*/
public function scopeForExternal($query)
{
return $query->whereNotNull('external_id');
}
/**
* @param Builder $query
* @return Builder
*/
public function scopeForNotExternal($query)
{
return $query->whereNull('external_id');
}
/**
* @param Builder $query
* @return Builder
*/
public function scopeForNotReserveCreate($query)
{
return $query->whereNull('reserve_create_at');
}
/**
* @param Builder $query
* @return Builder
*/
public function scopeForNotReserveUpdate($query)
{
return $query->whereNull('reserve_update_at');
}
/**
* @param Builder $query
* @return Builder
*/
public function scopeNeedUpdated($query)
{
return $query->whereNotNull('updated_need');
}
public function sitelink()
{
return $this->belongsTo(Sitelink::class, 'sitelink_id');
}
}
app/Models/Sitelink.php 0 → 100644
<?php
namespace App\Models;
use App\Models\Pivots\GoalSitelink;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection;
class Sitelink extends Model
{
use SoftDeletes;
protected $table = 'sitelinks';
protected $fillable = [
'external_id',
'links',
];
protected $casts = [
'links' => 'array',
];
/**
* @return Collection
*/
static public function getPropertiesWatch()
{
return collect([
'links',
]);
}
public function goalSitelinks()
{
return $this->hasMany(GoalSitelink::class, 'sitelink_id');
}
}
app/Models/Tokens.php
......@@ -5,6 +5,7 @@ namespace App\Models;
use App\Models\Pivots\DictionaryCampaign;
use App\Models\Pivots\GoalAdExtension;
use App\Models\Pivots\GoalNegativeKeywordSharedSet;
use App\Models\Pivots\GoalSitelink;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
......@@ -281,4 +282,14 @@ class Tokens extends Model
return $this->goalNegativeKeywordSharedSets()->forNotExternal()->forNotReserveCreate();
}
public function goalSitelinks()
{
return $this->hasMany(GoalSitelink::class, 'token_id');
}
public function goalSitelinksForNotExternalForNotReserveCreate()
{
return $this->goalSitelinks()->forNotExternal()->forNotReserveCreate();
}
}
app/Service/Requests/Direct/AddSitelinks.php 0 → 100644
<?php
namespace App\Service\Requests\Direct;
use App\Jobs\ProcessCallLimitedAPI;
use App\Models\Pivots\GoalSitelink;
use App\Models\Variable;
use App\Service\Contract\APIRequest;
use App\Service\Requests\DirectRequest;
use App\Service\StrReplaceByVariables;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Facades\Log;
class AddSitelinks extends DirectRequest
{
protected $max_count = 1000;
protected $timestamp;
/* @var Collection|GoalSitelink[] */
protected $goalSitelinks;
public function call($params = null)
{
$this->requestPrepare($params);
$process = new ProcessCallLimitedAPI($this);
dispatch($process)->onQueue('limits');
}
public function getObjectsCount()
{
return count($this->getParams()['AdExtensions']);
}
public function slice($maxObjects): ?APIRequest
{
$splinter = $this->sliceByKey($maxObjects, 'AdExtensions');
$splinter->putParams([
'goalSitelinks' => $this->goalSitelinks->slice($maxObjects)->values(),
]);
$this->putParams([
'goalSitelinks' => $this->goalSitelinks->slice(0, $maxObjects),
]);
return $splinter;
}
public function handle($response)
{
try {
if (!isset($response['result']['AddResults'])) {
return;
}
foreach ($response['result']['AddResults'] as $key => $add_result) {
$goalSitelink = $this->goalSitelinks->get($key);
if (!isset($add_result['Id'])) {
Log::debug("AddAdExtension, empty Id");
Log::debug($add_result);
Log::debug($this->getParams()['AdExtensions'][$key]);
$goalSitelink->update([
'reserve_create_at' => null,
]);
continue;
}
$external_id = (string)$add_result['Id'];
$goalSitelink->update([
'external_id' => $external_id,
'external_upload_at' => Carbon::now(),
'reserve_create_at' => null,
]);
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
public function failed()
{
GoalSitelink::whereIn('id', $this->goalSitelinks->pluck('id')->toArray())
->update([
'reserve_create_at' => null,
]);
}
public function putParams($params)
{
$this->goalSitelinks = $params['goalSitelinks'];
}
private function requestPrepare($params)
{
$this->setService('AdExtensions');
$this->setMethod('add');
$this->putParams($params);
$variables = Variable::all();
$lists = [];
$this->setParams([
'SitelinksSets' => $this->goalSitelinks->map(function (GoalSitelink $goalSitelink) use ($variables, &$lists) {
if (!isset($lists[$goalSitelink->dictionary_campaign_id])) {
$list = Variable::getListVariablesByDictionaryCampaign($goalSitelink->dictionary_campaign_id, $variables);
$lists[$goalSitelink->dictionary_campaign_id] = $list;
} else {
$list = $lists[$goalSitelink->dictionary_campaign_id];
}
return [
'Sitelinks' => array_map(function ($link) use ($list) {
return StrReplaceByVariables::getInstance($link, $list)->get();
}, $goalSitelink->sitelink->links),
];
})->toArray(),
]);
}
}
app/Service/Requests/Direct/GetSitelinks.php 0 → 100644
<?php
namespace App\Service\Requests\Direct;
use App\Jobs\ProcessCallLimitedAPI;
use App\Models\AdExtension;
use App\Models\AdGroup;
use App\Models\Advertisement;
use App\Models\Sitelink;
use App\Service\Contract\APIRequest;
use App\Service\Requests\DirectRequest;
use Carbon\Carbon;
use Illuminate\Support\Facades\Log;
class GetSitelinks extends DirectRequest
{
protected $max_count = -1;
protected $max_count_Ids = 10000;
function call($params = null)
{
$this->requestPrepare($params);
$process = new ProcessCallLimitedAPI($this);
dispatch($process)->onQueue('limits');
}
public function getObjectsCount()
{
$params = $this->getParams();
if (isset($params['SelectionCriteria']['Ids'])) {
return count($params['SelectionCriteria']['Ids']);
}
return -1;
}
public function slice($maxObjects): ?APIRequest
{
$params = $this->getParams();
if (isset($params['SelectionCriteria']['Ids'])) {
return $this->sliceByKey($maxObjects, ['SelectionCriteria', 'Ids']);
}
return null;
}
function handle($response)
{
try {
if (isset($response['result']['SitelinksSets'])) {
foreach ($response['result']['SitelinksSets'] as $sitelinks_sets) {
$external_id = (string)$sitelinks_sets['Id'];
if ($this->getToken()->isMain()) {
$data = [
'external_id' => $external_id,
'links' => $sitelinks_sets['Sitelinks'],
];
$sitelink = Sitelink::updateOrCreate([
'external_id' => $external_id
], $data);
if ($sitelink->wasChanged($sitelink::getPropertiesWatch()->toArray())) {
$sitelink->goalSitelinks()->forExternal()->update([
'updated_need' => Carbon::now(),
]);
}
} else {
//
}
}
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
private function requestPrepare($filter)
{
$this->setService('Sitelinks');
$this->setMethod('get');
$params = [
"FieldNames" => [
"Id", "Sitelinks",
],
"SitelinkFieldNames" => [
"Title", "Href", "Description",
],
];
if (isset($filter['Ids'])) {
$this->max_count = $this->max_count_Ids;
$params['SelectionCriteria'] = [
'Ids' => $filter['Ids'],
];
}
$this->setParams($params);
}
}
database/migrations/2021_07_07_090058_create_sitelinks_table.php 0 → 100644
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateSitelinksTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('sitelinks', function (Blueprint $table) {
$table->id();
$table->bigInteger('external_id');
$table->json('links')->nullable();
$table->softDeletes();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('sitelinks');
}
}
database/migrations/2021_07_07_090108_create_goal_sitelinks_table.php 0 → 100644
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateGoalSitelinksTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('goal_sitelinks', function (Blueprint $table) {
$table->id();
$table->bigInteger('external_id')->nullable();
$table->bigInteger('sitelink_id')->unsigned();
$table->bigInteger('token_id')->unsigned();
$table->timestamp('external_upload_at')->nullable();
$table->timestamp('external_updated_at')->nullable();
$table->timestamp('updated_need')->nullable();
$table->timestamp('reserve_create_at')->nullable();
$table->foreign('sitelink_id')->references('id')->on('sitelinks')
->cascadeOnDelete();
$table->foreign('token_id')->references('id')->on('tokens')
->cascadeOnDelete();
$table->softDeletes();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('goal_sitelinks');
}
}
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!