#20364 Загрузка и синхронизация расширений

app/Console/Commands/AdExtensionsLoad.php

+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Campaigns;
+use App\Models\Tokens;
+use App\Service\API\API;
+use App\Service\Requests\APIRequest;
+use Illuminate\Console\Command;
+
+class AdExtensionsLoad extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'adextensions:load';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Загрузка расширений';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     * @throws \Exception
+     */
+    public function handle()
+    {
+        $token = Tokens::where('type', Tokens::MAIN)->first();
+
+        if (!$token) {
+            throw new \Exception('Не найден токен блин');
+        }
+
+        $factory = APIRequest::getInstance(API::YANDEX);
+        $factory->setToken($token);
+
+        $factory->getRequest('AdExtensions', 'get')
+            ->call([
+                'ModifiedSince' => $token->check_changes_ad_extension,
+            ]);
+
+        return 0;
+    }
+}

app/Console/Commands/AdvertisementsAdd.php

@@ -58,6 +58,8 @@ class AdvertisementsAdd extends Command
 
             $goalAds = DB::table('goal_advertisements')
                 ->join('advertisements', 'goal_advertisements.advertisement_id', '=', 'advertisements.id')
+                ->leftJoin('goal_advertisement_goal_ad_extensions', 'goal_advertisements.id', '=', 'goal_advertisement_goal_ad_extensions.goal_advertisement_id')
+                ->leftJoin('goal_ad_extensions', 'goal_advertisements.id', '=', 'goal_ad_extensions.goal_ad_extension_id')
                 ->whereNull('advertisements.deleted_at')
                 ->whereNull('goal_advertisements.external_id')
                 ->whereNull('goal_advertisements.reserve_create_at')
@@ -77,6 +79,7 @@ class AdvertisementsAdd extends Command
                     'advertisements.v_card_id as v_card_id',
                     'advertisements.ad_image_hash as ad_image_hash',
                     'advertisements.site_link_set_id as site_link_set_id',
+                    DB::raw('JSON_OBJECTAGG(goal_ad_extensions.external_id) as ad_extension_ids'),
                     'advertisements.ad_extensions as ad_extensions',
                     'advertisements.video_extension as video_extension',
                     'advertisements.price_extension as price_extension',

app/Console/Commands/AdvertisementsUpdate.php

@@ -56,6 +56,8 @@ class AdvertisementsUpdate extends Command
 
             $goalAds = DB::table('goal_advertisements')
                 ->join('advertisements', 'goal_advertisements.advertisement_id', '=', 'advertisements.id')
+                ->leftJoin('goal_advertisement_goal_ad_extensions', 'goal_advertisements.id', '=', 'goal_advertisement_goal_ad_extensions.goal_advertisement_id')
+                ->leftJoin('goal_ad_extensions', 'goal_advertisements.id', '=', 'goal_ad_extensions.goal_ad_extension_id')
                 ->whereNull('advertisements.deleted_at')
                 ->whereNull('goal_advertisements.reserve_update_at')
                 ->whereNotNull('goal_advertisements.updated_need')
@@ -74,13 +76,14 @@ class AdvertisementsUpdate extends Command
                     'advertisements.v_card_id as v_card_id',
                     'advertisements.ad_image_hash as ad_image_hash',
                     'advertisements.site_link_set_id as site_link_set_id',
-                    'advertisements.ad_extensions as ad_extensions',
+                    DB::raw("JSON_OBJECTAGG('AdExtensionId', goal_ad_extensions.external_id, 'Operation', 'SET') as ad_extensions"),
                     'advertisements.video_extension as video_extension',
                     'advertisements.price_extension as price_extension',
                     'advertisements.turbo_page_id as turbo_page_id',
                     'advertisements.business_id as business_id',
                     'advertisements.prefer_v_card_over_business as prefer_v_card_over_business',
                 ])
+                ->groupBy('goal_advertisements.id')
                 ->get();
 
             foreach (array_chunk($goalAds->pluck('id')->toArray(), 1000) as $items){

app/Console/Commands/DictionaryCampaignsSyncByCampaign.php

@@ -9,6 +9,7 @@ use App\Models\Keyword;
 use App\Models\Pivots\DictionaryCampaign;
 use App\Models\Pivots\GoalAdGroup;
 use App\Models\Pivots\GoalKeyword;
+use App\Models\Tokens;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\DB;
 
@@ -131,6 +132,16 @@ class DictionaryCampaignsSyncByCampaign extends Command
                          LEFT JOIN goal_keywords gk on k.id = gk.keyword_id AND gk.goal_ad_group_id=gag.id
                     WHERE gk.keyword_id is null AND k.deleted_at is null
                             ");
+ 
+        //грузим расширения которых по какой то причне нет в целевых.
+        DB::insert("
+                INSERT INTO goal_ad_extensions(ad_extension_id, token_id, created_at, updated_at)
+                    SELECT ae.id, t.id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
+                    FROM ad_extensions ae
+                        INNER JOIN tokens t on t.type != '" . Tokens::MAIN . "'
+                        LEFT JOIN goal_ad_extensions gae on ae.id = gae.ad_extension_id and t.id = gae.token_id
+                    WHERE gae.ad_extension_id is null AND gae.deleted_at is null
+                            ");
 
         //грузим объявления которых по какой то причне нет в целевых.
         DB::insert("
@@ -141,7 +152,7 @@ class DictionaryCampaignsSyncByCampaign extends Command
                          INNER JOIN ad_groups ag on ad.ad_group_id = ag.id
                          INNER JOIN goal_ad_groups gag on ag.id = gag.ad_group_id
                          LEFT JOIN goal_advertisements gad on ad.id = gad.advertisement_id AND gad.goal_ad_group_id=gag.id
-                    WHERE gad.advertisement_id is null
+                    WHERE gad.advertisement_id is null and ad.campaign_id is not null
                             ");
 
         //грузим объявления которых по какой то причне нет в целевых.

app/Console/Kernel.php

@@ -2,6 +2,7 @@
 
 namespace App\Console;
 
+use App\Console\Commands\AdExtensionsLoad;
 use App\Console\Commands\AdGroupsAdd;
 use App\Console\Commands\AdGroupsLoadUpdated;
 use App\Console\Commands\AdGroupsUpdate;
@@ -62,6 +63,8 @@ class Kernel extends ConsoleKernel
         $schedule->command(AdGroupsLoadUpdated::class)->hourlyAt(20);
         $schedule->command(AdGroupsLoadKeywords::class)->hourlyAt(30);
 
+        $schedule->command(AdExtensionsLoad::class)->hourlyAt(25);
+
         $schedule->command(AdGroupsAdd::class)->hourlyAt(30);
         $schedule->command(AdGroupsUpdate::class)->hourlyAt(30);
 

app/Http/Controllers/CampaignVariablesController.php

@@ -112,7 +112,7 @@ class CampaignVariablesController extends Controller
     public function editVariable(Variable $variable)
     {
         $variable->update(Request::validate([
-            'name' => $this->rule_variable_name,
+            'name' => "{$this->rule_variable_name},{$variable->getKey()}",
             'default_value' => $this->rule_default_value,
         ]));
 

app/Models/AdExtension.php

+<?php
+
+namespace App\Models;
+
+use App\Models\Pivots\GoalAdExtension;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\SoftDeletes;
+use Illuminate\Support\Collection;
+
+class AdExtension extends Model
+{
+    use SoftDeletes;
+
+    const TYPE_CALLOUT = 'CALLOUT';
+    const TYPE_UNKNOWN = 'UNKNOWN';
+
+    const STATE_ON = 'ON';
+    const STATE_DELETED = 'DELETED';
+    const STATE_UNKNOWN = 'UNKNOWN';
+
+    const STATUS_ACCEPTED = 'ACCEPTED';
+    const STATUS_DRAFT = 'DRAFT';
+    const STATUS_MODERATION = 'MODERATION';
+    const STATUS_REJECTED = 'REJECTED';
+    const STATUS_UNKNOWN = 'UNKNOWN';
+
+
+    protected $fillable = [
+        'external_id',
+        'callout_text',
+        'associated',
+        'type',
+        'state',
+        'status',
+        'status_clarification',
+    ];
+
+    protected $casts = [
+        'associated' => 'boolean',
+    ];
+
+    /**
+     * @return Collection
+     */
+    static public function getPropertiesWatch()
+    {
+        return collect([
+            'callout_text',
+        ]);
+    }
+
+    public function goalAdExtensions()
+    {
+        return $this->hasMany(GoalAdExtension::class, 'ad_extension_id');
+    }
+
+}

app/Models/Pivots/GoalAdExtension.php

+<?php
+
+namespace App\Models\Pivots;
+
+use App\Models\AdExtension;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Relations\Pivot;
+use Illuminate\Database\Eloquent\SoftDeletes;
+
+class GoalAdExtension extends Pivot
+{
+    use SoftDeletes;
+
+    protected $table = 'goal_ad_extensions';
+
+    protected $fillable = [
+        'external_id',
+        'ad_extension_id',
+        'external_upload_at',
+        'external_updated_at',
+        'updated_need',
+        'reserve_create_at',
+        'reserve_update_at',
+    ];
+
+    protected $casts = [
+        'external_upload_at' => 'datetime',
+        'external_updated_at' => 'datetime',
+        'updated_need' => 'datetime',
+        'reserve_create_at' => 'datetime',
+        'reserve_update_at' => 'datetime',
+    ];
+
+    public $incrementing = true;
+
+    static public function getWithPivot()
+    {
+        return [
+            'id',
+            'external_id',
+            'ad_extension_id',
+            'external_upload_at',
+            'external_updated_at',
+            'updated_need',
+            'reserve_create_at',
+            'reserve_update_at',
+        ];
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForExternal($query)
+    {
+        return $query->whereNotNull('external_id');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForNotExternal($query)
+    {
+        return $query->whereNull('external_id');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForNotReserveCreate($query)
+    {
+        return $query->whereNull('reserve_create_at');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeForNotReserveUpdate($query)
+    {
+        return $query->whereNull('reserve_update_at');
+    }
+
+    /**
+     * @param Builder $query
+     * @return Builder
+     */
+    public function scopeNeedUpdated($query)
+    {
+        return $query->whereNotNull('updated_need');
+    }
+
+    public function adExtension()
+    {
+        return $this->belongsTo(AdExtension::class, 'ad_extension_id');
+    }
+
+}

app/Models/Tokens.php

@@ -22,6 +22,7 @@ use Illuminate\Database\Eloquent\Model;
  * @property \Illuminate\Support\Carbon $check_changes_campaign_at
  * @property \Illuminate\Support\Carbon|null $check_changes_ad_group
  * @property \Illuminate\Support\Carbon $check_changes_ad_group_at
+ * @property \Illuminate\Support\Carbon $check_changes_ad_extension
  * @property \Illuminate\Support\Carbon|null $created_at
  * @property \Illuminate\Support\Carbon|null $updated_at
  * @property int $limit
@@ -94,6 +95,7 @@ class Tokens extends Model
         'check_changes' => 'datetime',
         'check_changes_campaign' => 'datetime',
         'check_changes_ad_group' => 'datetime',
+        'check_changes_ad_extension' => 'datetime',
     ];
 
     public function getCheckChangesAtAttribute()

app/Service/Requests/Direct/AddAds.php

@@ -151,10 +151,8 @@ class AddAds extends DirectRequest
                     $data['TextAd']['SitelinkSetId'] = $goalAdvertisement->site_link_set_id;
                 }
 
-                if ($ad_extensions = @json_decode($goalAdvertisement->ad_extensions, true)) {
-                    $data['TextAd']['AdExtensionIds'] = array_map(function ($ad_extension) {
-                        return $ad_extension['AdExtensionId'];
-                    }, $ad_extensions);
+                if ($ad_extension_ids = array_filter(@json_decode($goalAdvertisement->ad_extension_ids, true))) {
+                    $data['TextAd']['AdExtensionIds'] = $ad_extension_ids;
                 }
 
                 if ($video_extension = @json_decode($goalAdvertisement->video_extension, true)) {

app/Service/Requests/Direct/GetAdExtensions.php

+<?php
+
+namespace App\Service\Requests\Direct;
+
+use App\Jobs\ProcessCallLimitedAPI;
+use App\Models\AdExtension;
+use App\Models\AdGroup;
+use App\Models\Advertisement;
+use App\Service\Contract\APIRequest;
+use App\Service\Requests\DirectRequest;
+use Carbon\Carbon;
+use Illuminate\Support\Facades\Log;
+
+class GetAdExtensions extends DirectRequest
+{
+    protected $max_count = -1;
+    protected $max_count_Ids = 10000;
+
+    function call($params = null)
+    {
+        $this->requestPrepare($params);
+        $process = new ProcessCallLimitedAPI($this);
+        dispatch($process)->onQueue('limits');
+    }
+
+    public function getObjectsCount()
+    {
+        $params = $this->getParams();
+        if (isset($params['SelectionCriteria']['Ids'])) {
+            return count($params['SelectionCriteria']['Ids']);
+        }
+        return -1;
+    }
+
+    public function slice($maxObjects): ?APIRequest
+    {
+        $params = $this->getParams();
+
+        if (isset($params['SelectionCriteria']['Ids'])) {
+            return $this->sliceByKey($maxObjects, ['SelectionCriteria', 'Ids']);
+        }
+
+        return null;
+    }
+
+    function handle($response)
+    {
+        try {
+            dd($response);
+            if (isset($response['result']['AdExtensions'])) {
+                foreach ($response['result']['AdExtensions'] as $ad_extension) {
+
+                    $external_id = (string)$ad_extension['Id'];
+
+                    if ($this->getToken()->isMain()) {
+
+                        $data = [
+                            'external_id' => $external_id,
+                            'callout_text' => $ad_extension['Callout']['CalloutText'],
+                            'associated' => $ad_extension['Associated'] === 'YES',
+                            'state' => $ad_extension['State'],
+                            'status' => $ad_extension['Status'],
+                            'status_clarification' => $ad_extension['StatusClarification'],
+                        ];
+
+                        $adExtension = AdExtension::updateOrCreate([
+                            'external_id' => $external_id
+                        ], $data);
+
+                        if ($adExtension->wasChanged($adExtension::getPropertiesWatch()->toArray())) {
+                            $adExtension->goalAdExtensions()->forExternal()->update([
+                                'updated_need' => Carbon::now(),
+                            ]);
+                        }
+
+                    } else {
+                        //
+                    }
+
+                }
+            }
+
+        } catch (\Exception $e) {
+            Log::debug($e);
+            throw $e;
+        }
+    }
+
+    private function requestPrepare($filter)
+    {
+        $this->setService('AdExtensions');
+        $this->setMethod('get');
+        $params = [
+            "SelectionCriteria" => [
+                "Types" => [
+                    AdExtension::TYPE_CALLOUT,
+                ],
+            ],
+            "FieldNames" => [
+                "Id", "Type", "Status",
+                "StatusClarification", "Associated",
+            ],
+            "CalloutFieldNames" => [
+                "CalloutText",
+            ],
+        ];
+        if (isset($filter['Ids'])) {
+
+            $this->max_count = $this->max_count_Ids;
+
+            $params['SelectionCriteria']['Ids'] = $filter['Ids'];
+        }
+        if (isset($filter['ModifiedSince']) && $filter['ModifiedSince']) {
+
+            $modified_since = $filter['ModifiedSince'];
+
+            if ($modified_since instanceof Carbon) {
+                $modified_since = $modified_since->toIso8601ZuluString();
+            }
+
+            $params['SelectionCriteria']['ModifiedSince'] = $modified_since;
+        }
+        $this->setParams($params);
+    }
+}

app/Service/Requests/Direct/GetAds.php

@@ -134,6 +134,14 @@ class GetAds extends DirectRequest
                             'business_id' => $ad['TextAd']['BusinessId'],
                             'prefer_v_card_over_business' => isset($ad['TextAd']['PreferVCardOverBusiness']) ? $ad['TextAd']['PreferVCardOverBusiness'] === 'YES' : null,
                         ]);
+
+                        foreach ($ad['TextAd']['AdExtensions'] as $ad_extensions_data) {
+
+                            /*
+                             * TODO: #20364 Загрузка и синхронизация расширений
+                            */
+
+                        }
                     }
 
                     $advertisement = Advertisement::updateOrCreate([

app/Service/Requests/Direct/UpdateAds.php

@@ -136,14 +136,9 @@ class UpdateAds extends DirectRequest
                     $data['TextAd']['SitelinkSetId'] = $goalAdvertisement->site_link_set_id;
                 }
 
-                if ($ad_extensions = @json_decode($goalAdvertisement->ad_extensions, true)) {
+                if ($ad_extensions = array_filter(@json_decode($goalAdvertisement->ad_extensions, true))) {
                     $data['TextAd']['CalloutSetting'] = [
-                        'AdExtensions' => array_map(function ($ad_extension) {
-                            return [
-                                'AdExtensionId' => $ad_extension['AdExtensionId'],
-                                'Operation' => 'SET',
-                            ];
-                        }, $ad_extensions)
+                        'AdExtensions' => $ad_extensions,
                     ];
                 }
 

database/migrations/2021_06_30_120819_create_ad_extensions_table.php

+<?php
+
+use App\Models\AdExtension;
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateAdExtensionsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('ad_extensions', function (Blueprint $table) {
+            $table->dropColumn('ad_extensions')->nullable();
+        });
+
+        Schema::create('ad_extensions', function (Blueprint $table) {
+            $table->id();
+
+            $table->bigInteger('external_id');
+            $table->string('callout_text');
+            $table->boolean('associated');
+            $table->enum('type', [
+                AdExtension::TYPE_CALLOUT,
+                AdExtension::TYPE_UNKNOWN,
+            ]);
+            $table->enum('state', [
+                AdExtension::STATE_ON,
+                AdExtension::STATE_DELETED,
+                AdExtension::STATE_UNKNOWN,
+            ]);
+            $table->enum('status', [
+                AdExtension::STATUS_ACCEPTED,
+                AdExtension::STATUS_DRAFT,
+                AdExtension::STATUS_DRAFT,
+                AdExtension::STATUS_MODERATION,
+                AdExtension::STATUS_REJECTED,
+                AdExtension::STATUS_UNKNOWN,
+            ]);
+            $table->string('status_clarification');
+
+            $table->softDeletes();
+            $table->timestamps();
+        });
+
+        Schema::table('tokens', function (Blueprint $table) {
+            $table->timestamp('check_changes_ad_extension')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('ad_extensions');
+    }
+}

database/migrations/2021_07_01_123052_create_advertisement_ad_extensions_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateAdvertisementAdExtensionsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('advertisement_ad_extensions', function (Blueprint $table) {
+            $table->id();
+
+            $table->bigInteger('advertisement_id')->unsigned();
+            $table->bigInteger('ad_extension_id')->unsigned();
+
+            $table->foreign('advertisement_id')->references('id')->on('advertisements')
+                ->cascadeOnDelete();
+            $table->foreign('ad_extension_id')->references('id')->on('ad_extensions')
+                ->cascadeOnDelete();
+
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('advertisement_ad_extensions');
+    }
+}

database/migrations/2021_07_01_141445_create_goal_ad_extensions_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateGoalAdExtensionsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('goal_ad_extensions', function (Blueprint $table) {
+            $table->id();
+
+            $table->id();
+
+            $table->bigInteger('external_id')->nullable();
+            $table->bigInteger('ad_extension_id')->unsigned();
+            $table->bigInteger('token_id')->unsigned();
+            $table->timestamp('external_upload_at')->nullable();
+            $table->timestamp('external_updated_at')->nullable();
+            $table->timestamp('updated_need')->nullable();
+
+            $table->timestamps();
+
+            $table->foreign('ad_extension_id')->references('id')->on('ad_extensions')
+                ->cascadeOnDelete();
+            $table->foreign('token_id')->references('id')->on('tokens')
+                ->cascadeOnDelete();
+
+            $table->softDeletes();
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('goal_ad_extensions');
+    }
+}

database/migrations/2021_07_01_154153_create_goal_advertisement_goal_ad_extensions_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateGoalAdvertisementGoalAdExtensionsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('goal_advertisement_goal_ad_extensions', function (Blueprint $table) {
+            $table->id();
+
+            $table->id();
+
+            $table->bigInteger('goal_advertisement_id')->unsigned();
+            $table->bigInteger('goal_ad_extension_id')->unsigned();
+
+            $table->foreign('goal_advertisement_id')->references('id')->on('goal_advertisements')
+                ->cascadeOnDelete();
+            $table->foreign('goal_ad_extension_id')->references('id')->on('goal_ad_extensions')
+                ->cascadeOnDelete();
+
+            $table->timestamps();
+
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('goal_advertisement_goal_ad_extensions');
+    }
+}