From 7409cfbea9a41457e356ad01bd5ee943a782d6e6 Mon Sep 17 00:00:00 2001 From: Jon Gilkison Date: Tue, 12 Sep 2017 03:55:38 +0700 Subject: [PATCH 1/3] Added rekognizer import tool. --- classes/tasks/ilab-recognizer-process.php | 116 ++++++++++++++ classes/tasks/wp-async-request.php | 12 +- classes/tasks/wp-background-process.php | 7 + .../ilab-media-rekognition-tool.php | 125 ++++++++++++++- classes/tools/s3/ilab-media-s3-tool.php | 3 +- .../rekognizer/ilab-rekognizer-processor.php | 150 ++++++++++++++++++ 6 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 classes/tasks/ilab-recognizer-process.php create mode 100644 views/rekognizer/ilab-rekognizer-processor.php diff --git a/classes/tasks/ilab-recognizer-process.php b/classes/tasks/ilab-recognizer-process.php new file mode 100644 index 00000000..9261c6d6 --- /dev/null +++ b/classes/tasks/ilab-recognizer-process.php @@ -0,0 +1,116 @@ +shouldHandle()) { + ILabMediaToolLogger::info('Task cancelled', $item); + return false; + } + + $index = $item['index']; + $post_id = $item['post']; + + update_option('ilab_rekognizer_current', $index+1); + $data = wp_get_attachment_metadata($post_id); + + if (empty($data)) { + ILabMediaToolLogger::info('Missing metadata', $item); + return false; + } + + if (!isset($data['s3'])) { + ILabMediaToolLogger::info('Missing s3 metadata', $item); + return false; + } + + $fileName = basename($data['file']); + update_option('ilab_rekognizer_current_file', $fileName); + + + $rekognizerTool = ILabMediaToolsManager::instance()->tools['rekognition']; + $data = $rekognizerTool->processImageMeta($post_id, $data); + wp_update_attachment_metadata($post_id, $data); + + return false; + } + + public function dispatch() { + ILabMediaToolLogger::info('Task dispatch'); + parent::dispatch(); + } + + protected function complete() { + ILabMediaToolLogger::info('Task complete'); + delete_option('ilab_rekognizer_status'); + delete_option('ilab_rekognizer_total_count'); + delete_option('ilab_rekognizer_current'); + delete_option('ilab_rekognizer_current_file'); + parent::complete(); + } + + public function cancel_process() { + ILabMediaToolLogger::info('Cancel process'); + + parent::cancel_process(); + + delete_option('ilab_rekognizer_status'); + delete_option('ilab_rekognizer_total_count'); + delete_option('ilab_rekognizer_current'); + delete_option('ilab_rekognizer_current_file'); + } + + public static function cancelAll() { + ILabMediaToolLogger::info('Cancel all processes'); + + wp_clear_scheduled_hook('wp_ilab_rekognizer_import_process_cron'); + + global $wpdb; + + $res = $wpdb->get_results("select * from {$wpdb->options} where option_name like 'wp_ilab_rekognizer_import_process_batch_%'"); + foreach($res as $batch) { + ILabMediaToolLogger::info("Deleting batch {$batch->option_name}"); + delete_option($batch->option_name); + } + + delete_option('ilab_rekognizer_status'); + delete_option('ilab_rekognizer_total_count'); + delete_option('ilab_rekognizer_current'); + delete_option('ilab_rekognizer_current_file'); + + ILabMediaToolLogger::info("Current cron", get_option('cron', [])); + ILabMediaToolLogger::info('End cancel all processes'); + } +} diff --git a/classes/tasks/wp-async-request.php b/classes/tasks/wp-async-request.php index 97187111..b61daa21 100755 --- a/classes/tasks/wp-async-request.php +++ b/classes/tasks/wp-async-request.php @@ -2,6 +2,8 @@ if (!defined('ABSPATH')) { header('Location: /'); die; } +require_once(ILAB_CLASSES_DIR.'/utils/ilab-media-tool-logger.php'); + /** * WP Async Request * @@ -87,6 +89,8 @@ public function dispatch() { $url = add_query_arg( $this->get_query_args(), $this->get_query_url() ); $args = $this->get_post_args(); + ILabMediaToolLogger::info("Background dispatching $url", $args); + return wp_remote_post( esc_url_raw( $url ), $args ); } @@ -144,7 +148,13 @@ protected function get_post_args() { * Check for correct nonce and pass to handler. */ public function maybe_handle() { - check_ajax_referer( $this->identifier, 'nonce' ); + $check = check_ajax_referer( $this->identifier, 'nonce', false ); + + ILabMediaToolLogger::info("Maybe handle, check nonce: ".(($check) ? 'true' : 'false')); + + if (!$check) { + wp_die(); + } $this->handle(); diff --git a/classes/tasks/wp-background-process.php b/classes/tasks/wp-background-process.php index 821b4399..f039abd4 100755 --- a/classes/tasks/wp-background-process.php +++ b/classes/tasks/wp-background-process.php @@ -2,6 +2,7 @@ if (!defined('ABSPATH')) { header('Location: /'); die; } require_once('wp-async-request.php'); +require_once(ILAB_CLASSES_DIR.'/utils/ilab-media-tool-logger.php'); /** * WP Background Process @@ -104,6 +105,7 @@ public function save() { $key = $this->generate_key(); if ( ! empty( $this->data ) ) { + ILabMediaToolLogger::info("Saving queue: $key", $this->data); update_site_option( $key, $this->data ); } @@ -230,6 +232,7 @@ protected function is_process_running() { * defined in the time_exceeded() method. */ protected function lock_process() { + ILabMediaToolLogger::info("Locking process {$this->identifier}"); $this->start_time = time(); // Set start time of current process. $lock_duration = ( property_exists( $this, 'queue_lock_time' ) ) ? $this->queue_lock_time : 60; // 1 minute @@ -246,6 +249,7 @@ protected function lock_process() { * @return $this */ protected function unlock_process() { + ILabMediaToolLogger::info("Unlocking process {$this->identifier}"); delete_site_transient( $this->identifier . '_process_lock' ); return $this; @@ -304,8 +308,11 @@ protected function handle() { do { $batch = $this->get_batch(); + ILabMediaToolLogger::info("Processing Batch", $batch); foreach ( $batch->data as $key => $value ) { if ($this->shouldHandle()) { + ILabMediaToolLogger::info("Running task", $value); + $task = $this->task( $value ); if ( false !== $task ) { diff --git a/classes/tools/rekognition/ilab-media-rekognition-tool.php b/classes/tools/rekognition/ilab-media-rekognition-tool.php index d09e7b11..1991ab81 100644 --- a/classes/tools/rekognition/ilab-media-rekognition-tool.php +++ b/classes/tools/rekognition/ilab-media-rekognition-tool.php @@ -14,6 +14,7 @@ if (!defined('ABSPATH')) { header('Location: /'); die; } require_once(ILAB_CLASSES_DIR.'/ilab-media-tool-base.php'); +require_once(ILAB_CLASSES_DIR.'/tasks/ilab-recognizer-process.php'); require_once(ILAB_CLASSES_DIR.'/utils/ilab-media-tool-logger.php'); if (file_exists(ILAB_VENDOR_DIR.'/autoload.php')) { @@ -44,6 +45,8 @@ class ILabMediaRekognitionTool extends ILabMediaToolBase { public function __construct($toolName, $toolInfo, $toolManager) { parent::__construct($toolName, $toolInfo, $toolManager); + new ILABRekognizerProcess(); + $this->key = $this->getOption('ilab-media-s3-access-key', 'ILAB_AWS_S3_ACCESS_KEY'); $this->secret = $this->getOption('ilab-media-s3-secret', 'ILAB_AWS_S3_ACCESS_SECRET'); @@ -85,8 +88,6 @@ public function __construct($toolName, $toolInfo, $toolManager) { $taxes[] = $this->detectCelebritiesTax; } - - add_action( 'init' , function() use ($taxes) { foreach($taxes as $tax) { if (in_array($tax, ['post_tag', 'category'])) { @@ -100,6 +101,10 @@ public function __construct($toolName, $toolInfo, $toolManager) { if (is_admin()) { $this->setupAdmin(); + + add_action('wp_ajax_ilab_rekognizer_process_images', [$this,'processImages']); + add_action('wp_ajax_ilab_rekognizer_process_progress', [$this,'processProgress']); + add_action('wp_ajax_ilab_rekognizer_cancel_process', [$this,'cancelProcessMedia']); } } @@ -123,6 +128,14 @@ public function enabled() { return ($this->detectLabels || $this->detectFaces || $this->detectExplicit || $this->detectCelebrities); } + public function registerMenu($top_menu_slug) { + parent::registerMenu($top_menu_slug); // TODO: Change the autogenerated stub + + if ($this->enabled()) { + add_submenu_page( $top_menu_slug, 'Rekognizer Importer', 'Rekognizer Importer', 'manage_options', 'media-tools-rekognizer-importer', [$this,'renderImporter']); + } + } + public function processImageMeta($postID, $meta) { if (!$this->enabled()) { return $meta; @@ -376,4 +389,112 @@ public function attachmentTaxonomies() { return $taxonomies; } + + public function renderImporter() { + $enabled = $this->enabled(); + + $shouldCancel = get_option('ilab_rekognizer_should_cancel', false); + $status = get_option('ilab_rekognizer_status', false); + $total = get_option('ilab_rekognizer_total_count', 0); + $current = get_option('ilab_rekognizer_current', 1); + $currentFile = get_option('ilab_rekognizer_current_file', ''); + + if ($total == 0) { + $attachments = get_posts([ + 'post_type'=> 'attachment', + 'posts_per_page' => -1 + ]); + + $total = count($attachments); + } + + $progress = 0; + + if ($total > 0) { + $progress = ($current / $total) * 100; + } + + echo ILabMediaToolView::render_view('rekognizer/ilab-rekognizer-processor.php',[ + 'status' => ($status) ? 'running' : 'idle', + 'total' => $total, + 'progress' => $progress, + 'current' => $current, + 'currentFile' => $currentFile, + 'enabled' => $enabled, + 'shouldCancel' => $shouldCancel + ]); + } + + public function processImages() { + global $wpdb; + + $sql = <<posts} posts + where + posts.post_type = 'attachment' + and + posts.post_status = 'inherit' + and + posts.post_mime_type in ('image/jpeg', 'image/jpg', 'image/png') + and + posts.ID in (select post_id from wp_postmeta where meta_key = '_wp_attachment_metadata' and meta_value like '%s:2:"s3";a:%'); +SQL; + + $rows = $wpdb->get_results($sql); + + $posts = []; + foreach($rows as $row) { + $posts[] = $row->ID; + } + + if (count($posts) > 0) { + update_option('ilab_rekognizer_status', true); + update_option('ilab_rekognizer_total_count', count($posts)); + update_option('ilab_rekognizer_current', 1); + update_option('ilab_rekognizer_should_cancel', false); + + $process = new ILABRekognizerProcess(); + + for($i = 0; $i < count($posts); ++$i) { + $process->push_to_queue(['index' => $i, 'post' => $posts[$i]]); + } + + $process->save(); + $process->dispatch(); + } else { + delete_option('ilab_rekognizer_status'); + } + + header('Content-type: application/json'); + echo '{"status":"running"}'; + die; + } + + public function processProgress() { + $shouldCancel = get_option('ilab_rekognizer_should_cancel', false); + $status = get_option('ilab_rekognizer_status', false); + $total = get_option('ilab_rekognizer_total_count', 0); + $current = get_option('ilab_rekognizer_current', 0); + $currentFile = get_option('ilab_rekognizer_current_file', ''); + + header('Content-type: application/json'); + echo json_encode([ + 'status' => ($status) ? 'running' : 'idle', + 'total' => (int)$total, + 'current' => (int)$current, + 'currentFile' => $currentFile, + 'shouldCancel' => $shouldCancel + ]); + die; + } + + public function cancelProcessMedia() { + update_option('ilab_rekognizer_should_cancel', 1); + ILABRekognizerProcess::cancelAll(); + + return json_response(['status'=>'ok']); + } } \ No newline at end of file diff --git a/classes/tools/s3/ilab-media-s3-tool.php b/classes/tools/s3/ilab-media-s3-tool.php index 4ea7f296..e1ecd304 100644 --- a/classes/tools/s3/ilab-media-s3-tool.php +++ b/classes/tools/s3/ilab-media-s3-tool.php @@ -118,8 +118,9 @@ public function __construct($toolName, $toolInfo, $toolManager) public function registerMenu($top_menu_slug) { parent::registerMenu($top_menu_slug); // TODO: Change the autogenerated stub - if (!$this->settingsError) + if (!$this->settingsError && $this->enabled()) { add_submenu_page( $top_menu_slug, 'S3 Importer', 'S3 Importer', 'manage_options', 'media-tools-s3-importer', [$this,'renderImporter']); + } } public function s3enabled() { diff --git a/views/rekognizer/ilab-rekognizer-processor.php b/views/rekognizer/ilab-rekognizer-processor.php new file mode 100644 index 00000000..0daeaa55 --- /dev/null +++ b/views/rekognizer/ilab-rekognizer-processor.php @@ -0,0 +1,150 @@ + +
+

Rekognizer Importer

+
+

This tool will run any images that have already been uploaded to S3 through Amazon Rekognizer.

+

Depending on the number of items you have, this could take anywhere from a minute to several hours. This process runs in the background until it's finished. Once you've started the process, please check this page for progress.

+

Note:

+
    +
  1. You MUST HAVE Rekognizer enabled and working in Tools Settings before running this task.
    2. +
+
+ + Process Images + + Please enable Rekognizer before using this tool. + +
+
+
+
+

Cancelling ... This may take a minute ...

+

Rekognizer importer is currently running. Processing '{{$currentFile}}' ({{$current}} of {{$total}}).

+
+
+
+
+ +
+ +
From a233672bd5c8a8a5a46e5d757ad9f387f2027107 Mon Sep 17 00:00:00 2001 From: Jon Gilkison Date: Tue, 12 Sep 2017 07:21:40 +0700 Subject: [PATCH 2/3] Fixes for the rekognizer process. --- classes/tasks/ilab-recognizer-process.php | 1 - classes/tasks/wp-background-process.php | 23 +++++++-- .../ilab-media-rekognition-tool.php | 50 +++++++++++++++++++ 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/classes/tasks/ilab-recognizer-process.php b/classes/tasks/ilab-recognizer-process.php index 9261c6d6..5f9bd643 100644 --- a/classes/tasks/ilab-recognizer-process.php +++ b/classes/tasks/ilab-recognizer-process.php @@ -29,7 +29,6 @@ class ILABRekognizerProcess extends ILAB_WP_Background_Process { protected function shouldHandle() { $result = !get_option('ilab_rekognizer_should_cancel', false); - ILabMediaToolLogger::info("Should cancel: ".(($result) ? 'true' : 'false')); return $result; } diff --git a/classes/tasks/wp-background-process.php b/classes/tasks/wp-background-process.php index f039abd4..0e8229dd 100755 --- a/classes/tasks/wp-background-process.php +++ b/classes/tasks/wp-background-process.php @@ -305,10 +305,18 @@ protected function shouldHandle() { protected function handle() { $this->lock_process(); + $shouldCancelAll = false; + do { $batch = $this->get_batch(); - ILabMediaToolLogger::info("Processing Batch", $batch); + if ((count($batch->data) == 0) || (!$this->shouldHandle())) { + $shouldCancelAll = true; + ILabMediaToolLogger::info("No batch to process", $batch); + break; + } + + ILabMediaToolLogger::info("Processing Batch", $batch->data); foreach ( $batch->data as $key => $value ) { if ($this->shouldHandle()) { ILabMediaToolLogger::info("Running task", $value); @@ -338,11 +346,15 @@ protected function handle() { $this->unlock_process(); - // Start next batch or complete process. - if ( ! $this->is_queue_empty() ) { - $this->dispatch(); - } else { + if ($shouldCancelAll) { $this->complete(); + static::cancelAll(); + } else { + if ( ! $this->is_queue_empty() ) { + $this->dispatch(); + } else { + $this->complete(); + } } wp_die(); @@ -516,5 +528,6 @@ public function cancel_process() { */ abstract public function task( $item ); + abstract public static function cancelAll(); } } diff --git a/classes/tools/rekognition/ilab-media-rekognition-tool.php b/classes/tools/rekognition/ilab-media-rekognition-tool.php index 1991ab81..49be62a3 100644 --- a/classes/tools/rekognition/ilab-media-rekognition-tool.php +++ b/classes/tools/rekognition/ilab-media-rekognition-tool.php @@ -105,6 +105,56 @@ public function __construct($toolName, $toolInfo, $toolManager) { add_action('wp_ajax_ilab_rekognizer_process_images', [$this,'processImages']); add_action('wp_ajax_ilab_rekognizer_process_progress', [$this,'processProgress']); add_action('wp_ajax_ilab_rekognizer_cancel_process', [$this,'cancelProcessMedia']); + + add_action('admin_init',function(){ + if ($this->enabled()) { + add_filter('bulk_actions-upload', function($actions){ + $actions['ilab_rekognizer_process'] = 'Process with Rekognizer'; + return $actions; + }); + + add_filter('handle_bulk_actions-upload', function($redirect_to, $action_name, $post_ids) { + if ('ilab_rekognizer_process' === $action_name) { + $posts_to_import = []; + if (count($post_ids) > 0) { + foreach($post_ids as $post_id) { + $meta = wp_get_attachment_metadata($post_id); + if (!empty($meta) && !isset($meta['s3'])) { + continue; + } + + $mime = get_post_mime_type($post_id); + if (!in_array($mime, ['image/jpeg', 'image/jpg', 'image/png'])) { + continue; + } + + $posts_to_import[] = $post_id; + } + } + + if (count($posts_to_import) > 0) { + update_option('ilab_rekognizer_status', true); + update_option('ilab_rekognizer_total_count', count($posts_to_import)); + update_option('ilab_rekognizer_current', 1); + update_option('ilab_rekognizer_should_cancel', false); + + $process = new ILABRekognizerProcess(); + + for($i = 0; $i < count($posts_to_import); ++$i) { + $process->push_to_queue(['index' => $i, 'post' => $posts_to_import[$i]]); + } + + $process->save(); + $process->dispatch(); + + return 'admin.php?page=media-tools-rekognizer-importer'; + } + } + + return $redirect_to; + }, 1000, 3); + } + }); } } From 26c16cea5d19917b783af0f69d68b129618ef5c6 Mon Sep 17 00:00:00 2001 From: Jon Gilkison Date: Tue, 12 Sep 2017 07:21:48 +0700 Subject: [PATCH 3/3] S3 icon overlay --- classes/tools/s3/ilab-media-s3-tool.php | 47 +++++++++++++++++++++++++ config/s3.config.php | 12 +++++++ public/img/ilab-s3-logo.svg | 23 ++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 public/img/ilab-s3-logo.svg diff --git a/classes/tools/s3/ilab-media-s3-tool.php b/classes/tools/s3/ilab-media-s3-tool.php index e1ecd304..80fcdd60 100644 --- a/classes/tools/s3/ilab-media-s3-tool.php +++ b/classes/tools/s3/ilab-media-s3-tool.php @@ -51,6 +51,8 @@ class ILabMediaS3Tool extends ILabMediaToolBase { private $skipUpdate = false; + private $displayBadges = true; + public function __construct($toolName, $toolInfo, $toolManager) { parent::__construct($toolName, $toolInfo, $toolManager); @@ -65,6 +67,7 @@ public function __construct($toolName, $toolInfo, $toolManager) $this->deleteFromS3 = $this->getOption('ilab-media-s3-delete-from-s3'); $this->prefixFormat = $this->getOption('ilab-media-s3-prefix', ''); $this->uploadDocs = $this->getOption('ilab-media-s3-upload-documents', null, true); + $this->displayBadges = $this->getOption('ilab-media-s3-display-s3-badge', null, true); $this->privacy = $this->getOption('ilab-media-s3-privacy', null, "public-read"); if (!in_array($this->privacy, ['public-read', 'authenticated-read'])) { $this->displayAdminNotice('error', "Your AWS S3 settings are incorrect. The ACL '{$this->privacy}' is not valid. Defaulting to 'public-read'."); @@ -113,6 +116,8 @@ public function __construct($toolName, $toolInfo, $toolManager) add_action('wp_ajax_ilab_s3_import_progress', [$this,'importProgress']); add_action('wp_ajax_ilab_s3_cancel_import', [$this,'cancelImportMedia']); } + + $this->hookMediaGrid(); } public function registerMenu($top_menu_slug) { @@ -1417,4 +1422,46 @@ public function customEndpointIsGoogle() { public function accessKey() { return $this->key; } + + protected function hookMediaGrid() { + if (!$this->displayBadges) { + return; + } + + add_action('admin_head', function(){ + ?> + + + + "text-field" ] ] + ], + "ilab-media-s3-display-settings" => [ + "title" => "Display Settings", + "description" => "", + "options" => [ + "ilab-media-s3-display-s3-badge" => [ + "title" => "Display S3 Icon", + "description" => "When this is selected, an S3 icon will be overlayed on items in the media library grid that are being served from S3.", + "type" => "checkbox", + "default" => true + ] + ] ] ] ] diff --git a/public/img/ilab-s3-logo.svg b/public/img/ilab-s3-logo.svg new file mode 100644 index 00000000..402af1d5 --- /dev/null +++ b/public/img/ilab-s3-logo.svg @@ -0,0 +1,23 @@ + + + + ilab-s3-logo + Created with Sketch. + + + + + \ No newline at end of file