#20434 Загрузка и синхронизация sitelinks

app/Console/Commands/DictionaryCampaignsSyncByCampaign.php

@@ -156,6 +156,16 @@ class DictionaryCampaignsSyncByCampaign extends Command
 
 
         //грузим расширения которых по какой то причне нет в целевых.
         //грузим расширения которых по какой то причне нет в целевых.
         DB::insert("
         DB::insert("
+                INSERT INTO goal_sitelinks(sitelink_id, token_id, created_at, updated_at)
+                    SELECT s.id, t.id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
+                    FROM sitelinks s
+                        INNER JOIN tokens t on t.type != '" . Tokens::MAIN . "'
+                        LEFT JOIN goal_sitelinks gs on s.id = gs.sitelink_id and t.id = gs.token_id
+                    WHERE gs.sitelink_id is null AND gs.deleted_at is null
+                            ");
+
+        //грузим наборы быстрых ссылок которых по какой то причне нет в целевых.
+        DB::insert("
                 INSERT INTO goal_ad_extensions(ad_extension_id, token_id, created_at, updated_at)
                 INSERT INTO goal_ad_extensions(ad_extension_id, token_id, created_at, updated_at)
                     SELECT ae.id, t.id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
                     SELECT ae.id, t.id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
                     FROM ad_extensions ae
                     FROM ad_extensions ae

app/Console/Commands/SitelinksAdd.php

+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Pivots\GoalAdExtension;
+use App\Models\Pivots\GoalSitelink;
+use App\Models\Tokens;
+use App\Service\API\API;
+use App\Service\Requests\APIRequest;
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Database\Eloquent\Relations\HasMany;
+
+class SitelinksAdd extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'sitelinks:add';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Добавление не созданных наборов быстрых ссылок с целевого аккаунта';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     * @throws \Exception
+     */
+    public function handle()
+    {
+        $tokens = Tokens::whereHas('goalSitelinksForNotExternalForNotReserveCreate.sitelink')
+            ->where('type', '!=', Tokens::MAIN)
+            ->get();
+
+        foreach ($tokens as $token) {
+            $token->load([
+                'goalSitelinksForNotExternalForNotReserveCreate' => function (HasMany $query) {
+                    return $query->has('sitelink');
+                },
+                'goalSitelinksForNotExternalForNotReserveCreate.sitelink',
+            ]);
+
+            $factory = APIRequest::getInstance(API::YANDEX);
+            $factory->setToken($token);
+
+            $goalSitelinks = $token->goalSitelinksForNotExternalForNotReserveCreate;
+
+            foreach (array_chunk($goalSitelinks->pluck('id')->toArray(), 1000) as $items) {
+                GoalSitelink::whereIn('id', $items)
+                    ->update([
+                        'reserve_create_at' => Carbon::now(),
+                    ]);
+            }
+
+            $factory->getRequest('Sitelinks', 'add')
+                ->call([
+                    'goalSitelinks' => $goalSitelinks,
+                ]);
+        }
+
+        return 0;
+    }
+}

app/Console/Commands/SitelinksLoad.php

+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Campaigns;
+use App\Models\Tokens;
+use App\Service\API\API;
+use App\Service\Requests\APIRequest;
+use Illuminate\Console\Command;
+
+class SitelinksLoad extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'sitelinks:load';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Загрузка наборы быстрых ссылок';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     * @throws \Exception
+     */
+    public function handle()
+    {
+        $token = Tokens::where('type', Tokens::MAIN)->first();
+
+        if (!$token) {
+            throw new \Exception('Не найден токен блин');
+        }
+
+        $factory = APIRequest::getInstance(API::YANDEX);
+        $factory->setToken($token);
+
+        $factory->getRequest('Sitelinks', 'get')
+            ->call();
+
+        return 0;
+    }
+}

app/Console/Kernel.php

@@ -25,6 +25,8 @@ use App\Console\Commands\KeywordsAdd;
 use App\Console\Commands\KeywordsDelete;
 use App\Console\Commands\KeywordsDelete;
 use App\Console\Commands\KeywordsUpdate;
 use App\Console\Commands\KeywordsUpdate;
 use App\Console\Commands\RefreshLimits;
 use App\Console\Commands\RefreshLimits;
+use App\Console\Commands\SitelinksAdd;
+use App\Console\Commands\SitelinksLoad;
 use App\Console\Commands\VCardsAdd;
 use App\Console\Commands\VCardsAdd;
 use App\Console\Commands\VCardsLoad;
 use App\Console\Commands\VCardsLoad;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Console\Scheduling\Schedule;
@@ -71,6 +73,9 @@ class Kernel extends ConsoleKernel
         $schedule->command(AdExtensionsLoad::class)->hourlyAt(25);
         $schedule->command(AdExtensionsLoad::class)->hourlyAt(25);
         $schedule->command(AdExtensionsAdd::class)->hourlyAt(30);
         $schedule->command(AdExtensionsAdd::class)->hourlyAt(30);
 
 
+        $schedule->command(SitelinksLoad::class)->hourlyAt(25);
+        $schedule->command(SitelinksAdd::class)->hourlyAt(30);
+
         $schedule->command(AdGroupsAdd::class)->hourlyAt(30);
         $schedule->command(AdGroupsAdd::class)->hourlyAt(30);
         $schedule->command(AdGroupsUpdate::class)->hourlyAt(30);
         $schedule->command(AdGroupsUpdate::class)->hourlyAt(30);
 
 

app/Models/Pivots/GoalSitelink.php

+<?php
+
+namespace App\Models\Pivots;
+
+use App\Models\Sitelink;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Relations\Pivot;
+use Illuminate\Database\Eloquent\SoftDeletes;
+
+class GoalSitelink extends Pivot
+{
+    use SoftDeletes;
+
+    protected $table = 'goal_sitelinks';
+
+    protected $fillable = [
+        'external_id',
+        'sitelink_id',
+        'token_id',
+        'external_upload_at',
+        'external_updated_at',
+        'updated_need',
+        'reserve_create_at',
+        'reserve_update_at',
+    ];
+
+    protected $casts = [
+        'external_upload_at' => 'datetime',
+        'external_updated_at' => 'datetime',
+        'updated_need' => 'datetime',
+        'reserve_create_at' => 'datetime',
+        'reserve_update_at' => 'datetime',
+    ];
+
+    public $incrementing = true;
+
+    static public function getWithPivot()
+    {
+        return [
+            'id',
+            'external_id',
+            'sitelink_id',
+            'token_id',
+            'external_upload_at',
+            'external_updated_at',
+            'updated_need',
+            'reserve_create_at',
+            'reserve_update_at',
+        ];
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForExternal($query)
+    {
+        return $query->whereNotNull('external_id');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForNotExternal($query)
+    {
+        return $query->whereNull('external_id');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForNotReserveCreate($query)
+    {
+        return $query->whereNull('reserve_create_at');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForNotReserveUpdate($query)
+    {
+        return $query->whereNull('reserve_update_at');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeNeedUpdated($query)
+    {
+        return $query->whereNotNull('updated_need');
+    }
+
+    public function sitelink()
+    {
+        return $this->belongsTo(Sitelink::class, 'sitelink_id');
+    }
+
+}

app/Models/Sitelink.php

+<?php
+
+namespace App\Models;
+
+use App\Models\Pivots\GoalSitelink;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\SoftDeletes;
+use Illuminate\Support\Collection;
+
+class Sitelink extends Model
+{
+    use SoftDeletes;
+
+    protected $table = 'sitelinks';
+
+    protected $fillable = [
+        'external_id',
+        'links',
+    ];
+
+    protected $casts = [
+        'links' => 'array',
+    ];
+
+    /**
+     * @return Collection
+     */
+    static public function getPropertiesWatch()
+    {
+        return collect([
+            'links',
+        ]);
+    }
+
+    public function goalSitelinks()
+    {
+        return $this->hasMany(GoalSitelink::class, 'sitelink_id');
+    }
+
+}

app/Models/Tokens.php

@@ -5,6 +5,7 @@ namespace App\Models;
 use App\Models\Pivots\DictionaryCampaign;
 use App\Models\Pivots\DictionaryCampaign;
 use App\Models\Pivots\GoalAdExtension;
 use App\Models\Pivots\GoalAdExtension;
 use App\Models\Pivots\GoalNegativeKeywordSharedSet;
 use App\Models\Pivots\GoalNegativeKeywordSharedSet;
+use App\Models\Pivots\GoalSitelink;
 use Carbon\Carbon;
 use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\Model;
@@ -281,4 +282,14 @@ class Tokens extends Model
         return $this->goalNegativeKeywordSharedSets()->forNotExternal()->forNotReserveCreate();
         return $this->goalNegativeKeywordSharedSets()->forNotExternal()->forNotReserveCreate();
     }
     }
 
 
+    public function goalSitelinks()
+    {
+        return $this->hasMany(GoalSitelink::class, 'token_id');
+    }
+
+    public function goalSitelinksForNotExternalForNotReserveCreate()
+    {
+        return $this->goalSitelinks()->forNotExternal()->forNotReserveCreate();
+    }
+
 }
 }

app/Service/Requests/Direct/AddSitelinks.php

+<?php
+
+namespace App\Service\Requests\Direct;
+
+use App\Jobs\ProcessCallLimitedAPI;
+use App\Models\Pivots\GoalSitelink;
+use App\Models\Variable;
+use App\Service\Contract\APIRequest;
+use App\Service\Requests\DirectRequest;
+use App\Service\StrReplaceByVariables;
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Support\Facades\Log;
+
+class AddSitelinks extends DirectRequest
+{
+    protected $max_count = 1000;
+
+    protected $timestamp;
+    /* @var Collection|GoalSitelink[] */
+    protected $goalSitelinks;
+
+    public function call($params = null)
+    {
+        $this->requestPrepare($params);
+        $process = new ProcessCallLimitedAPI($this);
+        dispatch($process)->onQueue('limits');
+    }
+
+    public function getObjectsCount()
+    {
+        return count($this->getParams()['AdExtensions']);
+    }
+
+    public function slice($maxObjects): ?APIRequest
+    {
+        $splinter = $this->sliceByKey($maxObjects, 'AdExtensions');
+
+        $splinter->putParams([
+            'goalSitelinks' => $this->goalSitelinks->slice($maxObjects)->values(),
+        ]);
+
+        $this->putParams([
+            'goalSitelinks' => $this->goalSitelinks->slice(0, $maxObjects),
+        ]);
+
+        return $splinter;
+    }
+
+    public function handle($response)
+    {
+        try {
+            if (!isset($response['result']['AddResults'])) {
+                return;
+            }
+
+            foreach ($response['result']['AddResults'] as $key => $add_result) {
+
+                $goalSitelink = $this->goalSitelinks->get($key);
+
+                if (!isset($add_result['Id'])) {
+                    Log::debug("AddAdExtension, empty Id");
+                    Log::debug($add_result);
+                    Log::debug($this->getParams()['AdExtensions'][$key]);
+
+                    $goalSitelink->update([
+                        'reserve_create_at' => null,
+                    ]);
+
+                    continue;
+                }
+
+                $external_id = (string)$add_result['Id'];
+
+                $goalSitelink->update([
+                    'external_id' => $external_id,
+                    'external_upload_at' => Carbon::now(),
+                    'reserve_create_at' => null,
+                ]);
+
+            }
+        } catch (\Exception $e) {
+            Log::debug($e);
+            throw $e;
+        }
+    }
+
+    public function failed()
+    {
+        GoalSitelink::whereIn('id', $this->goalSitelinks->pluck('id')->toArray())
+            ->update([
+                'reserve_create_at' => null,
+            ]);
+    }
+
+    public function putParams($params)
+    {
+        $this->goalSitelinks = $params['goalSitelinks'];
+    }
+
+    private function requestPrepare($params)
+    {
+        $this->setService('AdExtensions');
+        $this->setMethod('add');
+
+        $this->putParams($params);
+
+        $variables = Variable::all();
+
+        $lists = [];
+
+        $this->setParams([
+            'SitelinksSets' => $this->goalSitelinks->map(function (GoalSitelink $goalSitelink) use ($variables, &$lists) {
+
+                if (!isset($lists[$goalSitelink->dictionary_campaign_id])) {
+                    $list = Variable::getListVariablesByDictionaryCampaign($goalSitelink->dictionary_campaign_id, $variables);
+                    $lists[$goalSitelink->dictionary_campaign_id] = $list;
+                } else {
+                    $list = $lists[$goalSitelink->dictionary_campaign_id];
+                }
+
+                return [
+                    'Sitelinks' => array_map(function ($link) use ($list) {
+                        return StrReplaceByVariables::getInstance($link, $list)->get();
+                    }, $goalSitelink->sitelink->links),
+                ];
+
+            })->toArray(),
+        ]);
+    }
+
+}

app/Service/Requests/Direct/GetSitelinks.php

+<?php
+
+namespace App\Service\Requests\Direct;
+
+use App\Jobs\ProcessCallLimitedAPI;
+use App\Models\AdExtension;
+use App\Models\AdGroup;
+use App\Models\Advertisement;
+use App\Models\Sitelink;
+use App\Service\Contract\APIRequest;
+use App\Service\Requests\DirectRequest;
+use Carbon\Carbon;
+use Illuminate\Support\Facades\Log;
+
+class GetSitelinks extends DirectRequest
+{
+    protected $max_count = -1;
+    protected $max_count_Ids = 10000;
+
+    function call($params = null)
+    {
+        $this->requestPrepare($params);
+        $process = new ProcessCallLimitedAPI($this);
+        dispatch($process)->onQueue('limits');
+    }
+
+    public function getObjectsCount()
+    {
+        $params = $this->getParams();
+        if (isset($params['SelectionCriteria']['Ids'])) {
+            return count($params['SelectionCriteria']['Ids']);
+        }
+        return -1;
+    }
+
+    public function slice($maxObjects): ?APIRequest
+    {
+        $params = $this->getParams();
+
+        if (isset($params['SelectionCriteria']['Ids'])) {
+            return $this->sliceByKey($maxObjects, ['SelectionCriteria', 'Ids']);
+        }
+
+        return null;
+    }
+
+    function handle($response)
+    {
+        try {
+            if (isset($response['result']['SitelinksSets'])) {
+                foreach ($response['result']['SitelinksSets'] as $sitelinks_sets) {
+
+                    $external_id = (string)$sitelinks_sets['Id'];
+
+                    if ($this->getToken()->isMain()) {
+
+                        $data = [
+                            'external_id' => $external_id,
+                            'links' => $sitelinks_sets['Sitelinks'],
+                        ];
+
+                        $sitelink = Sitelink::updateOrCreate([
+                            'external_id' => $external_id
+                        ], $data);
+
+                        if ($sitelink->wasChanged($sitelink::getPropertiesWatch()->toArray())) {
+                            $sitelink->goalSitelinks()->forExternal()->update([
+                                'updated_need' => Carbon::now(),
+                            ]);
+                        }
+
+                    } else {
+                        //
+                    }
+
+                }
+            }
+
+        } catch (\Exception $e) {
+            Log::debug($e);
+            throw $e;
+        }
+    }
+
+    private function requestPrepare($filter)
+    {
+        $this->setService('Sitelinks');
+        $this->setMethod('get');
+        $params = [
+            "FieldNames" => [
+                "Id", "Sitelinks",
+            ],
+            "SitelinkFieldNames" => [
+                "Title", "Href", "Description",
+            ],
+        ];
+        if (isset($filter['Ids'])) {
+
+            $this->max_count = $this->max_count_Ids;
+
+            $params['SelectionCriteria'] = [
+                'Ids' => $filter['Ids'],
+            ];
+        }
+        $this->setParams($params);
+    }
+}

database/migrations/2021_07_07_090058_create_sitelinks_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateSitelinksTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('sitelinks', function (Blueprint $table) {
+            $table->id();
+
+            $table->bigInteger('external_id');
+            $table->json('links')->nullable();
+
+            $table->softDeletes();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('sitelinks');
+    }
+}

database/migrations/2021_07_07_090108_create_goal_sitelinks_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateGoalSitelinksTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('goal_sitelinks', function (Blueprint $table) {
+            $table->id();
+
+            $table->bigInteger('external_id')->nullable();
+            $table->bigInteger('sitelink_id')->unsigned();
+            $table->bigInteger('token_id')->unsigned();
+            $table->timestamp('external_upload_at')->nullable();
+            $table->timestamp('external_updated_at')->nullable();
+            $table->timestamp('updated_need')->nullable();
+            $table->timestamp('reserve_create_at')->nullable();
+
+            $table->foreign('sitelink_id')->references('id')->on('sitelinks')
+                ->cascadeOnDelete();
+            $table->foreign('token_id')->references('id')->on('tokens')
+                ->cascadeOnDelete();
+
+            $table->softDeletes();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('goal_sitelinks');
+    }
+}