From ee31932ad0fecd97a580a0f9888bb0a7729f2fdf Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 13:06:14 +0300 Subject: [PATCH 01/12] Add user references to tweets and metrics --- app/models/tweet.rb | 7 ++++ app/models/tweet_metric.rb | 3 ++ app/models/user.rb | 9 +++++ ...1008094036_add_user_id_to_tweet_metrics.rb | 18 +++++++++ .../20231008094222_add_user_id_to_tweets.rb | 18 +++++++++ db/schema.rb | 6 ++- lib/tasks/tweet_metrics_importer.rb | 5 +++ spec/factories/tweet_metrics.rb | 16 +++++--- spec/factories/tweets.rb | 7 ++++ spec/lib/tasks/tweet_metrics_importer_spec.rb | 8 +++- spec/models/tweet_metric_spec.rb | 7 ++++ spec/models/tweet_spec.rb | 20 +++++++++- spec/models/user_spec.rb | 40 +++++++++++++++++++ 13 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 db/migrate/20231008094036_add_user_id_to_tweet_metrics.rb create mode 100644 db/migrate/20231008094222_add_user_id_to_tweets.rb diff --git a/app/models/tweet.rb b/app/models/tweet.rb index af9b7d1..2b9551a 100644 --- a/app/models/tweet.rb +++ b/app/models/tweet.rb @@ -8,9 +8,16 @@ # url :string not null # created_at :datetime not null # updated_at :datetime not null +# user_id :bigint not null +# +# Foreign Keys +# +# fk_rails_... (user_id => users.id) # class Tweet < ApplicationRecord has_many :tweet_metrics, dependent: :destroy + belongs_to :user + validates :url, presence: true, uniqueness: true before_validation :set_author diff --git a/app/models/tweet_metric.rb b/app/models/tweet_metric.rb index 599dd1c..eb65db8 100644 --- a/app/models/tweet_metric.rb +++ b/app/models/tweet_metric.rb @@ -11,6 +11,7 @@ # created_at :datetime not null # updated_at :datetime not null # tweet_id :bigint not null +# user_id :bigint not null # # Indexes # @@ -20,7 +21,9 @@ # Foreign Keys # # fk_rails_... (tweet_id => tweets.id) +# fk_rails_... (user_id => users.id) # class TweetMetric < ApplicationRecord belongs_to :tweet + belongs_to :user end diff --git a/app/models/user.rb b/app/models/user.rb index 7e85ba3..035cb1d 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -16,8 +16,17 @@ class User < ApplicationRecord devise :database_authenticatable, :registerable, :rememberable, :validatable + + # TODO: Implement transfering tweets and tweet metrics to admin before deleting the user + has_many :tweets + has_many :tweet_metrics + validates :username, presence: true, uniqueness: {case_sensitive: false} + def admin? + username == Rails.application.credentials.admin_user.username + end + def email_required? false end diff --git a/db/migrate/20231008094036_add_user_id_to_tweet_metrics.rb b/db/migrate/20231008094036_add_user_id_to_tweet_metrics.rb new file mode 100644 index 0000000..675e184 --- /dev/null +++ b/db/migrate/20231008094036_add_user_id_to_tweet_metrics.rb @@ -0,0 +1,18 @@ +class AddUserIdToTweetMetrics < ActiveRecord::Migration[7.1] + def up + add_column :tweet_metrics, :user_id, :bigint + + first_user_id = execute("SELECT id FROM users ORDER BY created_at ASC LIMIT 1").first["id"] + + raise "No users found" if first_user_id.nil? + + execute("UPDATE tweet_metrics SET user_id = #{first_user_id}") + change_column_null :tweet_metrics, :user_id, false + add_foreign_key :tweet_metrics, :users + end + + def down + remove_foreign_key :tweet_metrics, :users + remove_column :tweet_metrics, :user_id + end +end diff --git a/db/migrate/20231008094222_add_user_id_to_tweets.rb b/db/migrate/20231008094222_add_user_id_to_tweets.rb new file mode 100644 index 0000000..c7a52c9 --- /dev/null +++ b/db/migrate/20231008094222_add_user_id_to_tweets.rb @@ -0,0 +1,18 @@ +class AddUserIdToTweets < ActiveRecord::Migration[7.1] + def up + add_column :tweets, :user_id, :bigint + + first_user_id = execute("SELECT id FROM users ORDER BY created_at ASC LIMIT 1").first["id"] + + raise "No users found" if first_user_id.nil? + + execute("UPDATE tweets SET user_id = #{first_user_id}") + change_column_null :tweets, :user_id, false + add_foreign_key :tweets, :users + end + + def down + remove_foreign_key :tweets, :users + remove_column :tweets, :user_id + end +end diff --git a/db/schema.rb b/db/schema.rb index 7ecf2e7..ea837ff 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2023_10_07_213927) do +ActiveRecord::Schema[7.1].define(version: 2023_10_08_094222) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -26,6 +26,7 @@ t.integer "views", default: 0 t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.bigint "user_id", null: false t.index ["tweet_id", "created_at"], name: "index_tweet_metrics_on_tweet_id_and_created_at", unique: true t.index ["tweet_id"], name: "index_tweet_metrics_on_tweet_id" end @@ -36,6 +37,7 @@ t.string "url", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.bigint "user_id", null: false end create_table "users", force: :cascade do |t| @@ -48,4 +50,6 @@ end add_foreign_key "tweet_metrics", "tweets" + add_foreign_key "tweet_metrics", "users" + add_foreign_key "tweets", "users" end diff --git a/lib/tasks/tweet_metrics_importer.rb b/lib/tasks/tweet_metrics_importer.rb index de4ae0f..1b3a947 100644 --- a/lib/tasks/tweet_metrics_importer.rb +++ b/lib/tasks/tweet_metrics_importer.rb @@ -21,6 +21,7 @@ def self.import_batch(batch) def self.transform_row(row) { + user_id: user.id, tweet_id: tweet.id, likes: row["count"], created_at: row["created_at"], @@ -28,6 +29,10 @@ def self.transform_row(row) } end + def self.user + @user ||= User.first + end + def self.tweet @tweet ||= Tweet.first end diff --git a/spec/factories/tweet_metrics.rb b/spec/factories/tweet_metrics.rb index 5bd7b22..1767577 100644 --- a/spec/factories/tweet_metrics.rb +++ b/spec/factories/tweet_metrics.rb @@ -11,6 +11,7 @@ # created_at :datetime not null # updated_at :datetime not null # tweet_id :bigint not null +# user_id :bigint not null # # Indexes # @@ -20,14 +21,17 @@ # Foreign Keys # # fk_rails_... (tweet_id => tweets.id) +# fk_rails_... (user_id => users.id) # FactoryBot.define do factory :tweet_metric do - tweet_id { nil } - replies { 1 } - reposts { 1 } - likes { 1 } - bookmarks { 1 } - views { 1 } + association :tweet + association :user + + replies { FFaker::Number.number(digits: 2) } + reposts { FFaker::Number.number(digits: 2) } + likes { FFaker::Number.number(digits: 3) } + bookmarks { FFaker::Number.number(digits: 1) } + views { FFaker::Number.number(digits: 4) } end end diff --git a/spec/factories/tweets.rb b/spec/factories/tweets.rb index cce2bbc..084b35e 100644 --- a/spec/factories/tweets.rb +++ b/spec/factories/tweets.rb @@ -8,9 +8,16 @@ # url :string not null # created_at :datetime not null # updated_at :datetime not null +# user_id :bigint not null +# +# Foreign Keys +# +# fk_rails_... (user_id => users.id) # FactoryBot.define do factory :tweet do + association :user + author { "P_Kallioniemi" } body { "In today's #vatniksoup, I'll introduce a South African-American(!) businessman and social media figure, Elon Musk" } url { "https://twitter.com/P_Kallioniemi/status/1674360288445964288" } diff --git a/spec/lib/tasks/tweet_metrics_importer_spec.rb b/spec/lib/tasks/tweet_metrics_importer_spec.rb index 561650b..6e0e9fc 100644 --- a/spec/lib/tasks/tweet_metrics_importer_spec.rb +++ b/spec/lib/tasks/tweet_metrics_importer_spec.rb @@ -2,6 +2,7 @@ require Rails.root.join("lib/tasks/tweet_metrics_importer") RSpec.describe Tasks::TweetMetricsImporter do + let!(:user) { create(:user) } let!(:tweet) { create(:tweet, id: 44) } let(:db_path) { "db/test-database.sqlite3" } let(:batch_size) { 10 } @@ -41,6 +42,7 @@ before do allow(described_class).to receive(:tweet).and_return(tweet) allow(described_class).to receive(:transform_row).and_return({ + user_id: user.id, tweet_id: 44, likes: 42, created_at: time, @@ -68,7 +70,7 @@ context "with existing records" do context "with a different time" do before do - create(:tweet_metric, tweet: tweet, likes: 42, created_at: time - 1.day, updated_at: time - 1.day) + create(:tweet_metric, tweet: tweet, user: user, likes: 42, created_at: time - 1.day, updated_at: time - 1.day) end it "creates a new record" do @@ -77,7 +79,7 @@ end context "with the different likes" do - let!(:existing_tweet_metric) { create(:tweet_metric, tweet: tweet, likes: 43, created_at: time, updated_at: time) } + let!(:existing_tweet_metric) { create(:tweet_metric, user: user, tweet: tweet, likes: 43, created_at: time, updated_at: time) } it "updates the existing record" do expect { subject }.to_not change { TweetMetric.count } @@ -88,6 +90,7 @@ context "with the same time and likes" do before do create(:tweet_metric, + user: user, tweet: tweet, likes: 42, created_at: time, @@ -109,6 +112,7 @@ "updated_at" => time } expect(described_class.transform_row(row)).to eq({ + user_id: user.id, tweet_id: 44, likes: 42, created_at: time, diff --git a/spec/models/tweet_metric_spec.rb b/spec/models/tweet_metric_spec.rb index 0400590..1bf47c1 100644 --- a/spec/models/tweet_metric_spec.rb +++ b/spec/models/tweet_metric_spec.rb @@ -11,6 +11,7 @@ # created_at :datetime not null # updated_at :datetime not null # tweet_id :bigint not null +# user_id :bigint not null # # Indexes # @@ -20,6 +21,7 @@ # Foreign Keys # # fk_rails_... (tweet_id => tweets.id) +# fk_rails_... (user_id => users.id) # require "rails_helper" @@ -29,5 +31,10 @@ association = described_class.reflect_on_association(:tweet) expect(association.macro).to eq(:belongs_to) end + + it "belongs to a user" do + association = described_class.reflect_on_association(:user) + expect(association.macro).to eq(:belongs_to) + end end end diff --git a/spec/models/tweet_spec.rb b/spec/models/tweet_spec.rb index d410028..de02092 100644 --- a/spec/models/tweet_spec.rb +++ b/spec/models/tweet_spec.rb @@ -8,11 +8,29 @@ # url :string not null # created_at :datetime not null # updated_at :datetime not null +# user_id :bigint not null +# +# Foreign Keys +# +# fk_rails_... (user_id => users.id) # require "rails_helper" RSpec.describe Tweet, type: :model do let(:tweet) { create(:tweet) } + let(:user) { create(:user) } + + describe "associations" do + it "belongs to a user" do + association = described_class.reflect_on_association(:user) + expect(association.macro).to eq(:belongs_to) + end + + it "has many tweet metrics" do + association = described_class.reflect_on_association(:tweet_metrics) + expect(association.macro).to eq(:has_many) + end + end describe "validations" do it "validates uniqueness of url" do @@ -31,7 +49,7 @@ describe "callbacks" do describe "#get_author" do it "sets the author from the url" do - new_tweet = Tweet.new(url: "https://twitter.com/P_Kallioniemi/status/1674360288445964288", author: nil) + new_tweet = Tweet.new(url: "https://twitter.com/P_Kallioniemi/status/1674360288445964288", author: nil, user: user) expect(new_tweet).to be_valid expect(new_tweet.author).to eq("P_Kallioniemi") end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index c28932f..29ae4ce 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -16,4 +16,44 @@ require "rails_helper" RSpec.describe User, type: :model do + describe "associations" do + it "has many tweets" do + association = described_class.reflect_on_association(:tweets) + expect(association.macro).to eq(:has_many) + end + + it "has many tweet metrics" do + association = described_class.reflect_on_association(:tweet_metrics) + expect(association.macro).to eq(:has_many) + end + end + + describe "validations" do + it "validates uniqueness of username" do + create(:user, username: "username") + new_user = User.new(username: "username") + expect(new_user).to_not be_valid + expect(new_user.errors.messages[:username]).to eq(["has already been taken"]) + end + + it "validates presence of username" do + new_user = User.new(username: nil) + expect(new_user).to_not be_valid + expect(new_user.errors.messages[:username]).to eq(["can't be blank"]) + end + end + + describe "instance methods" do + describe "#admin?" do + it "returns true if the user is the admin" do + admin_user = create(:user, username: Rails.application.credentials.admin_user.username, password: Rails.application.credentials.admin_user.password) + expect(admin_user.admin?).to eq(true) + end + + it "returns false if the user is not the admin" do + user = create(:user) + expect(user.admin?).to eq(false) + end + end + end end From 8539c675288b9da8122cf8594be8b07632e98b26 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 13:20:44 +0300 Subject: [PATCH 02/12] Add uuid to tweets --- app/models/tweet.rb | 7 ++++++- db/migrate/20231008101224_add_uuid_to_tweets.rb | 6 ++++++ db/schema.rb | 4 +++- spec/factories/tweets.rb | 6 ++++++ spec/models/tweet_spec.rb | 16 ++++++++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20231008101224_add_uuid_to_tweets.rb diff --git a/app/models/tweet.rb b/app/models/tweet.rb index 2b9551a..5777b6a 100644 --- a/app/models/tweet.rb +++ b/app/models/tweet.rb @@ -19,8 +19,9 @@ class Tweet < ApplicationRecord belongs_to :user validates :url, presence: true, uniqueness: true + validates :uuid, presence: true, uniqueness: true - before_validation :set_author + before_validation :set_author, :set_uuid def author_avatar_url # TODO: Implement a way to get the avatar url from the tweet @@ -69,4 +70,8 @@ def last_metric def set_author self.author = url&.split("/")&.fetch(3) end + + def set_uuid + self.uuid = SecureRandom.uuid if uuid.blank? + end end diff --git a/db/migrate/20231008101224_add_uuid_to_tweets.rb b/db/migrate/20231008101224_add_uuid_to_tweets.rb new file mode 100644 index 0000000..f94debb --- /dev/null +++ b/db/migrate/20231008101224_add_uuid_to_tweets.rb @@ -0,0 +1,6 @@ +class AddUuidToTweets < ActiveRecord::Migration[7.1] + def change + add_column :tweets, :uuid, :uuid, default: "gen_random_uuid()", null: false + add_index :tweets, :uuid, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index ea837ff..b6119e6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2023_10_08_094222) do +ActiveRecord::Schema[7.1].define(version: 2023_10_08_101224) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -38,6 +38,8 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.bigint "user_id", null: false + t.uuid "uuid", default: -> { "gen_random_uuid()" }, null: false + t.index ["uuid"], name: "index_tweets_on_uuid", unique: true end create_table "users", force: :cascade do |t| diff --git a/spec/factories/tweets.rb b/spec/factories/tweets.rb index 084b35e..5ee69d5 100644 --- a/spec/factories/tweets.rb +++ b/spec/factories/tweets.rb @@ -6,10 +6,15 @@ # author :string # body :text # url :string not null +# uuid :uuid not null # created_at :datetime not null # updated_at :datetime not null # user_id :bigint not null # +# Indexes +# +# index_tweets_on_uuid (uuid) UNIQUE +# # Foreign Keys # # fk_rails_... (user_id => users.id) @@ -21,5 +26,6 @@ author { "P_Kallioniemi" } body { "In today's #vatniksoup, I'll introduce a South African-American(!) businessman and social media figure, Elon Musk" } url { "https://twitter.com/P_Kallioniemi/status/1674360288445964288" } + uuid { FFaker::Guid.guid } end end diff --git a/spec/models/tweet_spec.rb b/spec/models/tweet_spec.rb index de02092..0a88ecd 100644 --- a/spec/models/tweet_spec.rb +++ b/spec/models/tweet_spec.rb @@ -6,10 +6,15 @@ # author :string # body :text # url :string not null +# uuid :uuid not null # created_at :datetime not null # updated_at :datetime not null # user_id :bigint not null # +# Indexes +# +# index_tweets_on_uuid (uuid) UNIQUE +# # Foreign Keys # # fk_rails_... (user_id => users.id) @@ -44,6 +49,12 @@ expect(new_tweet).to_not be_valid expect(new_tweet.errors.messages[:url]).to eq(["can't be blank"]) end + + it "validates uniqueness of uuid" do + new_tweet = Tweet.new(uuid: tweet.uuid) + expect(new_tweet).to_not be_valid + expect(new_tweet.errors.messages[:uuid]).to eq(["has already been taken"]) + end end describe "callbacks" do @@ -54,6 +65,11 @@ expect(new_tweet.author).to eq("P_Kallioniemi") end end + + it "sets the uuid" do + new_tweet = Tweet.create!(url: "https://twitter.com/P_Kallioniemi/status/1674360288445964288", uuid: nil, user: user) + expect(new_tweet.uuid).not_to be_nil + end end describe "instance methods" do From 15c8032e68e4415dcde8569ea393da3445969477 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 14:02:00 +0300 Subject: [PATCH 03/12] Add API endpoint to receive metrics --- Gemfile | 2 +- app/controllers/tweets_controller.rb | 30 ++++++++++++++ config/routes.rb | 4 +- spec/requests/tweets_spec.rb | 61 ++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 2101171..528775b 100644 --- a/Gemfile +++ b/Gemfile @@ -45,7 +45,6 @@ gem "sqlite3", "~> 1.6" group :development, :test do # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem gem "annotate", "~> 3.2" - gem "better_errors", "~> 2.10" gem "binding_of_caller", "~> 1.0" gem "debug", platforms: %i[mri mingw x64_mingw] gem "factory_bot_rails", "~> 6.2" @@ -60,6 +59,7 @@ group :development, :test do end group :development do + gem "better_errors", "~> 2.10" # Use console on exceptions pages [https://github.com/rails/web-console] gem "web-console" end diff --git a/app/controllers/tweets_controller.rb b/app/controllers/tweets_controller.rb index be11327..90afacd 100644 --- a/app/controllers/tweets_controller.rb +++ b/app/controllers/tweets_controller.rb @@ -1,4 +1,5 @@ class TweetsController < ApplicationController + skip_before_action :verify_authenticity_token, only: [:receive_metrics] # Skip CSRF protection for this API endpoint caches_action :index, :show, expires_in: 15.minutes def index @@ -20,4 +21,33 @@ def show end end end + + def receive_metrics + tweet = Tweet.find_by(uuid: uuid_param) + return render json: {error: "Tweet not found"}, status: :not_found unless tweet + + tweet.tweet_metrics.create!(metrics_params.merge(user: tweet.user)) + + tweet.update!(tweet_params) if tweet_params[:body].present? + + if tweet.body.blank? + render json: {command: :fetch_tweet_details} + else + render json: {message: :ok} + end + end + + private + + def uuid_param + params.require(:uuid) + end + + def metrics_params + params.permit(:likes, :reposts, :replies, :bookmarks, :views) + end + + def tweet_params + params.fetch(:tweet, {}).permit(:body) + end end diff --git a/config/routes.rb b/config/routes.rb index 874dd3f..78a26fb 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,8 @@ Rails.application.routes.draw do devise_for :users - resources :tweets, only: %i[index show] + resources :tweets, only: %i[index show] do + post "receive_metrics", on: :collection + end root "welcome#index" end diff --git a/spec/requests/tweets_spec.rb b/spec/requests/tweets_spec.rb index f1e0577..fc884db 100644 --- a/spec/requests/tweets_spec.rb +++ b/spec/requests/tweets_spec.rb @@ -31,6 +31,67 @@ end end + describe "POST /receive_metrics" do + subject { post "/tweets/receive_metrics", params: params } + + let(:metric_params) { {likes: 11, reposts: 22, replies: 33, bookmarks: 44, views: 55} } + let(:params) { {uuid: uuid}.merge(metric_params) } + let(:uuid) { tweet.uuid } + + context "when the tweet exists" do + it "creates a new tweet metric with the correct values" do + expect { subject }.to change { TweetMetric.count }.by(1) + expect(TweetMetric.last).to have_attributes( + tweet: tweet, + user: tweet.user, + likes: 11, + reposts: 22, + replies: 33, + bookmarks: 44, + views: 55 + ) + end + + it "returns a success message" do + subject + expect(response.body).to eq({message: :ok}.to_json) + end + end + + context "when the tweet does not exist" do + let(:uuid) { "non-existent-uuid" } + + it "returns an error message" do + subject + expect(response.body).to eq({error: "Tweet not found"}.to_json) + end + end + + context "when the tweet does not have a body" do + let(:tweet) { create(:tweet, body: nil) } + let(:uuid) { tweet.uuid } + + it "returns a fetch tweet details command" do + subject + expect(response.body).to eq({command: :fetch_tweet_details}.to_json) + end + + context "when tracker sends a body" do + let(:params) { {uuid: uuid, tweet: {body: "new body"}.merge(metric_params)} } + + it "updates the tweet body" do + subject + expect(tweet.reload.body).to eq("new body") + end + + it "returns a success message" do + subject + expect(response.body).to eq({message: :ok}.to_json) + end + end + end + end + # Test signup/login/logout since we don't have a separate controller for users describe "POST /users" do let(:user_params) { attributes_for(:user) } From 7b215ec46112ce6bbf675e1ed7d2855632b027d2 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 15:34:36 +0300 Subject: [PATCH 04/12] Tracker script [WIP] --- app/views/tweets/tracker.js.erb | 147 ++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 app/views/tweets/tracker.js.erb diff --git a/app/views/tweets/tracker.js.erb b/app/views/tweets/tracker.js.erb new file mode 100644 index 0000000..dd127dd --- /dev/null +++ b/app/views/tweets/tracker.js.erb @@ -0,0 +1,147 @@ +// ==UserScript== +// @name XTracker - tracking <%= @tweet.author %> #<%= @tweet.id %> +// @namespace http://tampermonkey.net/ +// @version 1.0 +// @description Track a specific tweet metrics +// @author https://github.com/hoblin/x-tracker +// @match <%= @tweet.url %> +// @grant GM_xmlhttpRequest +// @require https://code.jquery.com/jquery-3.6.0.min.js +// ==/UserScript== + +(function ($) { + "use strict"; + + const DEBUG = true; // Set to false to disable logging + const reportURL = "<%= @report_url %>" + const updateLimit = 10; // Number of updates before sending to metrics to the reportURL + + let labelFound = false; + let globalObserver; + let metrics = {likes: 0, reposts: 0, replies: 0, bookmarks: 0, views: 0}; + let updateCounter = 0; + + function log(...args) { + if (DEBUG) { + console.log(...args); + } + } + + function sendMetrics() { + // Send metrics to the server along with the UUID + // {uuid: "1234-5678-9012-3456", likes: 123, reposts: 456, replies: 789, bookmarks: 1011, views: 1213} + log("Sending metrics to reportURL:", metrics); + let data = JSON.stringify({ uuid: uuid, ...metrics }); + GM_xmlhttpRequest({ + method: "POST", + url: reportURL, + data: data, + headers: { + "Content-Type": "application/json", + }, + onload: function (response) { + log("Response from the server:", response); + }, + onerror: function (error) { + log("Error sending to the server:", error); + }, + }) + } + + function updateMetrics(targetNode) { + // On any metric change, we update the stored value and increment the counter of the metric updates. + // When the counter reaches the updateLimit, we send the metrics to the reportURL + const ariaLabel = $(targetNode).attr("aria-label"); + // extract the numbers from the aria-label + const numbers = ariaLabel.match(/\d+/g); + if (numbers) { + const replies = numbers[0]; + const reposts = numbers[1]; + const likes = numbers[2]; + const bookmarks = numbers[3]; + const views = numbers[4]; + const metrics = {likes, reposts, replies, bookmarks, views}; + log("Metrics updated:", metrics); + updateCounter += 1; + if (updateCounter >= updateLimit) { + log("Sending metrics to reportURL:", metrics); + sendMetrics(); + updateCounter = 0; + } + } + } + + function startObservingMetrics(targetNode) { + // Start observing the metrics label + log( + "Starting to observe metrics on label: ", + $(targetNode).attr("aria-label") + ); + // send the initial metrics + updateMetrics(targetNode); + const config = { attributes: true, childList: true, subtree: true }; + const callback = function (mutationsList, _observer) { + for (let mutation of mutationsList) { + if ( + mutation.type === "attributes" && + mutation.attributeName === "aria-label" + ) { + updateMetrics(mutation.target); + } + } + }; + + const observer = new MutationObserver(callback); + observer.observe(targetNode, config); + } + + function findLabel() { + log("Trying to find label"); + // look for the label with this attribute: + // aria-label="3 replies, 13 reposts, 110 likes, 1 bookmark, 5776 views" + // and start observing it + const targetNode = $('div[role="group"]').filter(function () { + const ariaLabel = $(this).attr("aria-label"); + return ( + ariaLabel && + typeof ariaLabel === "string" && + ariaLabel.indexOf("replies") > -1 && + ariaLabel.indexOf("reposts") > -1 && + ariaLabel.indexOf("likes") > -1 && + ariaLabel.indexOf("bookmark") > -1 && + ariaLabel.indexOf("views") > -1 + ); + })[0]; + + if (targetNode) { + log("Label found: ", $(targetNode).attr("aria-label")); + labelFound = true; + globalObserver.disconnect(); + startObservingMetrics(targetNode); + } + } + + function setupGlobalObserver() { + const config = { childList: true, subtree: true }; + const callback = function () { + if (!labelFound) { + findLabel(); + } else { + // If label is found, stop the global observer + globalObserver.disconnect(); + } + }; + + globalObserver = new MutationObserver(callback); + globalObserver.observe(document, config); + } + + $(document).ready(function () { + setupGlobalObserver(); + }); + + // Schedule a page refresh every minute (60,000 milliseconds) + setTimeout(function() { + window.location.reload(); + }, 60000); +})(jQuery); From 7dd424770ed339de4a6705009dfb7352fd87f763 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 18:41:18 +0300 Subject: [PATCH 05/12] Tracker --- app/controllers/tweets_controller.rb | 26 +++++--- app/decorators/tweet_decorator.rb | 4 +- app/views/tweets/show.html.slim | 10 +++ .../tweets/{tracker.js.erb => track.js.erb} | 66 +++++++++++++------ config/routes.rb | 6 +- 5 files changed, 77 insertions(+), 35 deletions(-) rename app/views/tweets/{tracker.js.erb => track.js.erb} (68%) diff --git a/app/controllers/tweets_controller.rb b/app/controllers/tweets_controller.rb index 90afacd..d3e9614 100644 --- a/app/controllers/tweets_controller.rb +++ b/app/controllers/tweets_controller.rb @@ -1,5 +1,5 @@ class TweetsController < ApplicationController - skip_before_action :verify_authenticity_token, only: [:receive_metrics] # Skip CSRF protection for this API endpoint + skip_forgery_protection only: %i[track receive_metrics] # Skip CSRF protection for Tampermonkey POST requests caches_action :index, :show, expires_in: 15.minutes def index @@ -23,12 +23,12 @@ def show end def receive_metrics - tweet = Tweet.find_by(uuid: uuid_param) + tweet = Tweet.find_by(uuid: tweet_params[:uuid]) return render json: {error: "Tweet not found"}, status: :not_found unless tweet tweet.tweet_metrics.create!(metrics_params.merge(user: tweet.user)) - tweet.update!(tweet_params) if tweet_params[:body].present? + tweet.update!(body: tweet_params[:body]) if tweet_params[:body].present? if tweet.body.blank? render json: {command: :fetch_tweet_details} @@ -37,17 +37,23 @@ def receive_metrics end end - private + # render js template using headers expected by Tampermonkey to install it + def track + @tweet = Tweet.find(params[:id]).decorate + @report_url = receive_metrics_url - def uuid_param - params.require(:uuid) + respond_to do |format| + format.js { render layout: false, content_type: "application/javascript" } + end end - def metrics_params - params.permit(:likes, :reposts, :replies, :bookmarks, :views) - end + private def tweet_params - params.fetch(:tweet, {}).permit(:body) + params.require(:tweet).permit(:uuid, :body) + end + + def metrics_params + params.require(:metrics).permit(:likes, :reposts, :replies, :bookmarks, :views) end end diff --git a/app/decorators/tweet_decorator.rb b/app/decorators/tweet_decorator.rb index 502fb3e..8d5570d 100644 --- a/app/decorators/tweet_decorator.rb +++ b/app/decorators/tweet_decorator.rb @@ -109,8 +109,8 @@ def combined_metrics_series likes_metric, replies_metric, reposts_metric, - bookmarks_metric, - views_metric + bookmarks_metric + # views_metric TODO: enable after getting rid of chartkick and switching to custom Y axis ] end diff --git a/app/views/tweets/show.html.slim b/app/views/tweets/show.html.slim index 746ed84..60b9630 100644 --- a/app/views/tweets/show.html.slim +++ b/app/views/tweets/show.html.slim @@ -27,6 +27,16 @@ section.hero.is-info | Comments: #{number_with_delimiter(@tweet.replies)} p | Views: #{number_with_delimiter(@tweet.views)} + .card + .card-content + .content + .columns.is-centered + .column.is-6 + .content + = link_to "Install tracker", track_user_js_path(@tweet), class: "button is-primary is-fullwidth is-large" + .column.is-6 + .content + = link_to "Start tracking", @tweet.url, class: "button is-primary is-fullwidth is-large", target: "_blank" hr .content .card diff --git a/app/views/tweets/tracker.js.erb b/app/views/tweets/track.js.erb similarity index 68% rename from app/views/tweets/tracker.js.erb rename to app/views/tweets/track.js.erb index dd127dd..8829877 100644 --- a/app/views/tweets/tracker.js.erb +++ b/app/views/tweets/track.js.erb @@ -12,9 +12,10 @@ (function ($) { "use strict"; - const DEBUG = true; // Set to false to disable logging + const DEBUG = false; // Set to false to disable logging const reportURL = "<%= @report_url %>" - const updateLimit = 10; // Number of updates before sending to metrics to the reportURL + const uuid = "<%= @tweet.uuid %>" + const updateLimit = 5; // Number of updates before sending to metrics to the reportURL let labelFound = false; let globalObserver; @@ -31,7 +32,8 @@ // Send metrics to the server along with the UUID // {uuid: "1234-5678-9012-3456", likes: 123, reposts: 456, replies: 789, bookmarks: 1011, views: 1213} log("Sending metrics to reportURL:", metrics); - let data = JSON.stringify({ uuid: uuid, ...metrics }); + let data = JSON.stringify({ tweet: { uuid: uuid }, metrics }); + log("Sending data to reportURL:", data); GM_xmlhttpRequest({ method: "POST", url: reportURL, @@ -48,23 +50,56 @@ }) } + function checkNode(node) { + const ariaLabel = $(node).attr("aria-label"); + return ( + ariaLabel && + typeof ariaLabel === "string" && + ariaLabel.indexOf("replies") > -1 && + ariaLabel.indexOf("reposts") > -1 && + ariaLabel.indexOf("likes") > -1 && + ariaLabel.indexOf("bookmark") > -1 && + ariaLabel.indexOf("views") > -1 + ); + } + function updateMetrics(targetNode) { + // make sure we have a valid targetNode + if (checkNode(targetNode) === false) { + return; + } // On any metric change, we update the stored value and increment the counter of the metric updates. // When the counter reaches the updateLimit, we send the metrics to the reportURL const ariaLabel = $(targetNode).attr("aria-label"); // extract the numbers from the aria-label const numbers = ariaLabel.match(/\d+/g); if (numbers) { - const replies = numbers[0]; - const reposts = numbers[1]; - const likes = numbers[2]; - const bookmarks = numbers[3]; - const views = numbers[4]; - const metrics = {likes, reposts, replies, bookmarks, views}; - log("Metrics updated:", metrics); + const replies = parseInt(numbers[0]); + const reposts = parseInt(numbers[1]); + const likes = parseInt(numbers[2]); + const bookmarks = parseInt(numbers[3]); + const views = parseInt(numbers[4]); + const newMetrics = { likes, reposts, replies, bookmarks, views }; + log("Metrics extracted from aria-label:", newMetrics); updateCounter += 1; + // send the metrics if any metric has changed from zero + if ( + (replies > 0 && metrics.replies === 0) || + (reposts > 0 && metrics.reposts === 0) || + (likes > 0 && metrics.likes === 0) || + (bookmarks > 0 && metrics.bookmarks === 0) || + (views > 0 && metrics.views === 0) + ) { + log("Sending first metrics to reportURL:", metrics); + metrics = newMetrics; + sendMetrics(); + updateCounter = 0; + } + + // send the metrics if the counter reaches the updateLimit if (updateCounter >= updateLimit) { log("Sending metrics to reportURL:", metrics); + metrics = newMetrics; sendMetrics(); updateCounter = 0; } @@ -101,16 +136,7 @@ // aria-label="3 replies, 13 reposts, 110 likes, 1 bookmark, 5776 views" // and start observing it const targetNode = $('div[role="group"]').filter(function () { - const ariaLabel = $(this).attr("aria-label"); - return ( - ariaLabel && - typeof ariaLabel === "string" && - ariaLabel.indexOf("replies") > -1 && - ariaLabel.indexOf("reposts") > -1 && - ariaLabel.indexOf("likes") > -1 && - ariaLabel.indexOf("bookmark") > -1 && - ariaLabel.indexOf("views") > -1 - ); + return checkNode(this); })[0]; if (targetNode) { diff --git a/config/routes.rb b/config/routes.rb index 78a26fb..cc5ecdf 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,8 +1,8 @@ Rails.application.routes.draw do devise_for :users - resources :tweets, only: %i[index show] do - post "receive_metrics", on: :collection - end + resources :tweets, only: %i[index show] + get "track/:id.user.js", to: "tweets#track", as: :track_user_js + post "receive_metrics", to: "tweets#receive_metrics", as: :receive_metrics root "welcome#index" end From 43dbeec81bab71025226936658173799223309e0 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 19:01:58 +0300 Subject: [PATCH 06/12] Check access for tracker sctipt --- app/controllers/application_controller.rb | 4 + app/controllers/tweets_controller.rb | 5 ++ app/views/tweets/show.html.slim | 21 ++--- spec/requests/tweets_spec.rb | 93 ++++++++++++++++++++++- 4 files changed, 110 insertions(+), 13 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index e91f867..a42a7a9 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -12,4 +12,8 @@ def configure_permitted_parameters def after_sign_in_path_for(resource) tweets_path end + + def record_not_found + render plain: "404 Not Found", status: :not_found + end end diff --git a/app/controllers/tweets_controller.rb b/app/controllers/tweets_controller.rb index d3e9614..8f066b6 100644 --- a/app/controllers/tweets_controller.rb +++ b/app/controllers/tweets_controller.rb @@ -20,6 +20,8 @@ def show render json: @tweet.tweet_metrics.to_json end end + rescue ActiveRecord::RecordNotFound + record_not_found end def receive_metrics @@ -42,6 +44,9 @@ def track @tweet = Tweet.find(params[:id]).decorate @report_url = receive_metrics_url + # TODO: replace with cancancan + return record_not_found unless user_signed_in? && current_user == @tweet.user + respond_to do |format| format.js { render layout: false, content_type: "application/javascript" } end diff --git a/app/views/tweets/show.html.slim b/app/views/tweets/show.html.slim index 60b9630..dc07a94 100644 --- a/app/views/tweets/show.html.slim +++ b/app/views/tweets/show.html.slim @@ -27,16 +27,17 @@ section.hero.is-info | Comments: #{number_with_delimiter(@tweet.replies)} p | Views: #{number_with_delimiter(@tweet.views)} - .card - .card-content - .content - .columns.is-centered - .column.is-6 - .content - = link_to "Install tracker", track_user_js_path(@tweet), class: "button is-primary is-fullwidth is-large" - .column.is-6 - .content - = link_to "Start tracking", @tweet.url, class: "button is-primary is-fullwidth is-large", target: "_blank" + - if user_signed_in? && @tweet.user == current_user + .card + .card-content + .content + .columns.is-centered + .column.is-6 + .content + = link_to "Install tracker", track_user_js_path(@tweet), class: "button is-primary is-fullwidth is-large" + .column.is-6 + .content + = link_to "Start tracking", @tweet.url, class: "button is-primary is-fullwidth is-large", target: "_blank" hr .content .card diff --git a/spec/requests/tweets_spec.rb b/spec/requests/tweets_spec.rb index fc884db..945de68 100644 --- a/spec/requests/tweets_spec.rb +++ b/spec/requests/tweets_spec.rb @@ -29,13 +29,100 @@ subject expect(response.body).to include(tweet.author) end + + context "when the tweet does not exist" do + subject { get "/tweets/999" } + + it "returns http not found" do + subject + expect(response).to have_http_status(:not_found) + end + end + + context "for a tweet owner" do + let(:user) { tweet.user } + + before do + sign_in user + end + + it "shows Install tracker link" do + subject + expect(response.body).to include("Install tracker") + end + + it "shows Start tracking link" do + subject + expect(response.body).to include("Start tracking") + end + end + + context "for a non-tweet owner" do + let(:user) { create(:user) } + + before do + sign_in user + end + + it "does not show Install tracker link" do + subject + expect(response.body).to_not include("Install tracker") + end + + it "does not show Start tracking link" do + subject + expect(response.body).to_not include("Start tracking") + end + end + end + + describe "GET /track/:id.user.js" do + subject { get "/track/#{tweet.id}.user.js" } + + context "for a tweet owner" do + let(:user) { tweet.user } + + before do + sign_in user + end + + it "returns http success" do + subject + expect(response).to have_http_status(:success) + end + + it "returns a script" do + subject + expect(response.body).to include("==UserScript==") + end + end + + context "for a non-tweet owner" do + let(:user) { create(:user) } + + before do + sign_in user + end + + it "returns http not found" do + subject + expect(response).to have_http_status(:not_found) + end + end + + context "for a non-logged in user" do + it "returns http not found" do + subject + expect(response).to have_http_status(:not_found) + end + end end describe "POST /receive_metrics" do - subject { post "/tweets/receive_metrics", params: params } + subject { post "/receive_metrics", params: params } let(:metric_params) { {likes: 11, reposts: 22, replies: 33, bookmarks: 44, views: 55} } - let(:params) { {uuid: uuid}.merge(metric_params) } + let(:params) { {tweet: {uuid: uuid}, metrics: metric_params} } let(:uuid) { tweet.uuid } context "when the tweet exists" do @@ -77,7 +164,7 @@ end context "when tracker sends a body" do - let(:params) { {uuid: uuid, tweet: {body: "new body"}.merge(metric_params)} } + let(:params) { {tweet: {uuid: uuid, body: "new body"}, metrics: metric_params} } it "updates the tweet body" do subject From 9269606f1fbbf07c1b113f95184288af11fb2784 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 23:20:59 +0300 Subject: [PATCH 07/12] Dynamic group chart resolution --- app/decorators/tweet_decorator.rb | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/app/decorators/tweet_decorator.rb b/app/decorators/tweet_decorator.rb index 8d5570d..7e435af 100644 --- a/app/decorators/tweet_decorator.rb +++ b/app/decorators/tweet_decorator.rb @@ -106,31 +106,36 @@ def chart_options def combined_metrics_series [ - likes_metric, - replies_metric, - reposts_metric, - bookmarks_metric + likes_metric(resolution), + replies_metric(resolution), + reposts_metric(resolution), + bookmarks_metric(resolution), # views_metric TODO: enable after getting rid of chartkick and switching to custom Y axis ] end - def likes_metric - @likes_metric ||= {name: "Likes", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: 5).maximum(:likes)} + def resolution + # resolution is 1 minute if we don't have many metrics and gradually increases up to 1 hour depending on the number of metrics + @resolution ||= object.tweet_metrics.count / 2000 + 1 end - def replies_metric - @replies_metric ||= {name: "Replies", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: 5).maximum(:replies)} + def likes_metric(minutes = 5) + @likes_metric ||= {name: "Likes", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: minutes).maximum(:likes)} end - def reposts_metric - @reposts_metric ||= {name: "Reposts", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: 5).maximum(:reposts)} + def replies_metric(minutes = 5) + @replies_metric ||= {name: "Replies", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: minutes).maximum(:replies)} end - def bookmarks_metric - @bookmarks_metric ||= {name: "Bookmarks", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: 5).maximum(:bookmarks)} + def reposts_metric(minutes = 5) + @reposts_metric ||= {name: "Reposts", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: minutes).maximum(:reposts)} end - def views_metric - @views_metric ||= {name: "Views", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: 5).maximum(:views)} + def bookmarks_metric(minutes = 5) + @bookmarks_metric ||= {name: "Bookmarks", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: minutes).maximum(:bookmarks)} + end + + def views_metric(minutes = 5) + @views_metric ||= {name: "Views", data: object.tweet_metrics.group_by_minute(:created_at, series: false, n: minutes).maximum(:views)} end end From 8a03599dd5bb67cd26934d21ca03b19bc186da69 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 23:22:35 +0300 Subject: [PATCH 08/12] Add tweet + better data tracking --- app/assets/stylesheets/style.scss | 2 +- app/controllers/tweets_controller.rb | 26 +++- app/models/tweet.rb | 16 ++- app/views/tweets/_form.html.slim | 11 ++ app/views/tweets/index.html.slim | 4 +- app/views/tweets/new.html.slim | 53 +++++++++ app/views/tweets/track.js.erb | 112 ++++++++++++------ config/routes.rb | 2 +- .../20231008184323_add_avatar_to_tweets.rb | 5 + db/schema.rb | 3 +- spec/models/tweet_spec.rb | 2 +- spec/requests/tweets_spec.rb | 18 ++- 12 files changed, 203 insertions(+), 51 deletions(-) create mode 100644 app/views/tweets/_form.html.slim create mode 100644 app/views/tweets/new.html.slim create mode 100644 db/migrate/20231008184323_add_avatar_to_tweets.rb diff --git a/app/assets/stylesheets/style.scss b/app/assets/stylesheets/style.scss index df1a01d..dccee95 100644 --- a/app/assets/stylesheets/style.scss +++ b/app/assets/stylesheets/style.scss @@ -18,7 +18,7 @@ h1.rotated { font-size: 3rem; } -.hero.is-info .has-dark-text .title { +.hero.is-info .has-dark-text .title, .hero.is-info .has-dark-text .subtitle { color: #000; } diff --git a/app/controllers/tweets_controller.rb b/app/controllers/tweets_controller.rb index 8f066b6..6b589a8 100644 --- a/app/controllers/tweets_controller.rb +++ b/app/controllers/tweets_controller.rb @@ -1,6 +1,7 @@ class TweetsController < ApplicationController skip_forgery_protection only: %i[track receive_metrics] # Skip CSRF protection for Tampermonkey POST requests caches_action :index, :show, expires_in: 15.minutes + before_action :authenticate_user!, only: %i[new create track] def index @tweets = Tweet.first(10).map(&:decorate) @@ -24,13 +25,30 @@ def show record_not_found end + def new + @tweet = Tweet.new + end + + def create + @tweet = current_user.tweets.new(create_tweet_params) + + if @tweet.save + redirect_to @tweet + else + respond_to do |format| + format.turbo_stream { render turbo_stream: turbo_stream.replace(@tweet, partial: "tweets/form", locals: {tweet: @tweet}) } + format.html { render :new } + end + end + end + def receive_metrics tweet = Tweet.find_by(uuid: tweet_params[:uuid]) return render json: {error: "Tweet not found"}, status: :not_found unless tweet tweet.tweet_metrics.create!(metrics_params.merge(user: tweet.user)) - tweet.update!(body: tweet_params[:body]) if tweet_params[:body].present? + tweet.update!(tweet_params.except(:uuid)) if tweet_params[:body].present? if tweet.body.blank? render json: {command: :fetch_tweet_details} @@ -54,8 +72,12 @@ def track private + def create_tweet_params + params.require(:tweet).permit(:url) + end + def tweet_params - params.require(:tweet).permit(:uuid, :body) + params.require(:tweet).permit(:uuid, :body, :avatar) end def metrics_params diff --git a/app/models/tweet.rb b/app/models/tweet.rb index 5777b6a..5ba5641 100644 --- a/app/models/tweet.rb +++ b/app/models/tweet.rb @@ -18,14 +18,13 @@ class Tweet < ApplicationRecord has_many :tweet_metrics, dependent: :destroy belongs_to :user - validates :url, presence: true, uniqueness: true + validates :url, presence: true, uniqueness: true, format: %r{https://(twitter|x).com/\w+/status/\d+} validates :uuid, presence: true, uniqueness: true before_validation :set_author, :set_uuid def author_avatar_url - # TODO: Implement a way to get the avatar url from the tweet - "https://pbs.twimg.com/profile_images/1678305940481753089/g751T5c__400x400.jpg" + avatar end def author_url @@ -61,6 +60,10 @@ def views last_metric&.views end + def match_url + url.sub(%r{https://(twitter|x).com}, "https://*").sub(/\?.*/, "*") + end + private def last_metric @@ -68,10 +71,17 @@ def last_metric end def set_author + return unless valid_url? + self.author = url&.split("/")&.fetch(3) end def set_uuid self.uuid = SecureRandom.uuid if uuid.blank? end + + def valid_url? + # Ensure the valid tweet url: https://twitter.com/P_Kallioniemi/status/1674360288445964288 + url&.match?(%r{https://(twitter|x).com/\w+/status/\d+}) + end end diff --git a/app/views/tweets/_form.html.slim b/app/views/tweets/_form.html.slim new file mode 100644 index 0000000..3ab4d64 --- /dev/null +++ b/app/views/tweets/_form.html.slim @@ -0,0 +1,11 @@ +#new_tweet + h3.has-text-centered.subtitle Add a New Tweet + = form_with model: tweet do |f| + .field.has-addons.is-large + .control.is-expanded.is-large + = f.text_field :url, class: "input is-large#{ ' is-danger' if tweet.errors[:url].any? }", placeholder: 'Enter Tweet URL' + .control + = f.submit 'Add Tweet', class: 'button is-primary is-large' + - tweet.errors.full_messages.each do |message| + p.help.is-danger.is-size-4 + = message diff --git a/app/views/tweets/index.html.slim b/app/views/tweets/index.html.slim index 626dbd1..2cbd065 100644 --- a/app/views/tweets/index.html.slim +++ b/app/views/tweets/index.html.slim @@ -6,6 +6,8 @@ section.hero.is-info .column.is-1 h1.rotated.has-text-grey Tracked Tweets .column.is-10 + .section + = link_to 'Add Tweet', new_tweet_path, class: 'button is-primary is-fullwidth is-large' - @tweets.each do |tweet| .card .card-content @@ -17,7 +19,7 @@ section.hero.is-info p class="title is-4" = link_to tweet.author_name, tweet.author_url, target: "_blank" p class="subtitle is-6" - = tweet.body.truncate(280) + = tweet.body&.truncate(280) .content = tweet.combined_chart footer.card-footer diff --git a/app/views/tweets/new.html.slim b/app/views/tweets/new.html.slim new file mode 100644 index 0000000..bb5c9d1 --- /dev/null +++ b/app/views/tweets/new.html.slim @@ -0,0 +1,53 @@ + section.hero.is-info.is-fullheight + .hero-head + = render "shared/navbar" + .hero-body + .container + .columns.is-centered + .column.is-10 + .box.has-dark-text + .content + h1.title.has-text-centered Before you start tracking + p.subtitle.has-text-centered + | X-Tracker provides real-time tweet metrics tracking right in your browser. Here's a brief on how it works: + ul + li + strong> Tampermonkey Extension: + | Our tracker operates through the Tampermonkey browser extension. It's a user script manager that allows the script to interact with the tweet page and send metrics to X-Tracker. + li + strong> Tracker Script: + | The tracker script does the magic. Once installed, it observes the tweet page, collects metrics, and sends them to X-Tracker, where you can visualize the data. + li + strong> Real-Time Tracking: + | Metrics are updated in real-time. You can start and stop tracking anytime, and analyze the tweet's performance right on X-Tracker. + p.has-text-centered.subtitle + | Ready to dive in? Follow the steps below: + + ol + li + strong Add a Tweet to X-Tracker: + ul + li Input the URL of the tweet you want to track in the form below and click "Submit." + li You'll be redirected to the tweet's page on X-Tracker. + li.mt-3 + strong Install the Tracker: + ul + li Click the "Install Tracker" button on the tweet's page on X-Tracker. A Tampermonkey window will pop up. + li Click "Install" on the Tampermonkey window to add the tracker script. + li.mt-3 + strong Start Tracking: + ul + li Now, click the "Start Tracking" button on the tweet's page on X-Tracker. + li The tweet URL will open in a new browser tab. + li A prompt will appear, asking for your permission to send data to X-Tracker. Click "Allow" to start tracking the tweet metrics. + li The tracking will commence, and you can view the metrics on the tweet's page on X-Tracker. + p.has-text-centered.subtitle + | Install the Tampermonkey extension on your browser to get started: + nav.buttons.is-centered + - %w(chrome firefox safari edge opera).each do |browser| + a.button.is-link.is-outlined.is-rounded href="https://www.tampermonkey.net/index.php?browser=#{browser}" target="_blank" + span.icon + i.fab class="fa-#{browser}" aria-hidden="true" + span= browser.capitalize + + = render partial: 'tweets/form', locals: { tweet: @tweet } diff --git a/app/views/tweets/track.js.erb b/app/views/tweets/track.js.erb index 8829877..fd17dc1 100644 --- a/app/views/tweets/track.js.erb +++ b/app/views/tweets/track.js.erb @@ -4,7 +4,7 @@ // @version 1.0 // @description Track a specific tweet metrics // @author https://github.com/hoblin/x-tracker -// @match <%= @tweet.url %> +// @match <%= @tweet.match_url %> // @grant GM_xmlhttpRequest // @require https://code.jquery.com/jquery-3.6.0.min.js // ==/UserScript== @@ -12,7 +12,7 @@ (function ($) { "use strict"; - const DEBUG = false; // Set to false to disable logging + const DEBUG = true; // Set to false to disable logging const reportURL = "<%= @report_url %>" const uuid = "<%= @tweet.uuid %>" const updateLimit = 5; // Number of updates before sending to metrics to the reportURL @@ -21,6 +21,7 @@ let globalObserver; let metrics = {likes: 0, reposts: 0, replies: 0, bookmarks: 0, views: 0}; let updateCounter = 0; + let tweetDataRequired = false; function log(...args) { if (DEBUG) { @@ -28,11 +29,33 @@ } } + function getTweetText() { + return $('#react-root').find('article').find('div[lang]').text(); + } + + function getAvatar() { + let avatar = null; + let counter = 0; + $('#react-root').find('img').each(function () { + let url = $(this).attr('src'); + if(url && url.indexOf('profile_images') > -1) { + counter += 1; + avatar = url; + if (counter > 1) { + return false; + } + } + }); + return avatar; + } + function sendMetrics() { - // Send metrics to the server along with the UUID - // {uuid: "1234-5678-9012-3456", likes: 123, reposts: 456, replies: 789, bookmarks: 1011, views: 1213} + console.log('Test avatar:', getAvatar()); log("Sending metrics to reportURL:", metrics); let data = JSON.stringify({ tweet: { uuid: uuid }, metrics }); + if (tweetDataRequired) { + data = JSON.stringify({ tweet: { uuid: uuid, body: getTweetText(), avatar: getAvatar() }, metrics }); + } log("Sending data to reportURL:", data); GM_xmlhttpRequest({ method: "POST", @@ -43,6 +66,12 @@ }, onload: function (response) { log("Response from the server:", response); + // if response contains "{command: :fetch_tweet_details}" + // we need to fetch the tweet text and avatar + if (response.responseText.indexOf("fetch_tweet_details") > -1) { + tweetDataRequired = true; + sendMetrics(); + } }, onerror: function (error) { log("Error sending to the server:", error); @@ -55,10 +84,7 @@ return ( ariaLabel && typeof ariaLabel === "string" && - ariaLabel.indexOf("replies") > -1 && - ariaLabel.indexOf("reposts") > -1 && ariaLabel.indexOf("likes") > -1 && - ariaLabel.indexOf("bookmark") > -1 && ariaLabel.indexOf("views") > -1 ); } @@ -68,42 +94,50 @@ if (checkNode(targetNode) === false) { return; } - // On any metric change, we update the stored value and increment the counter of the metric updates. - // When the counter reaches the updateLimit, we send the metrics to the reportURL + const ariaLabel = $(targetNode).attr("aria-label"); - // extract the numbers from the aria-label - const numbers = ariaLabel.match(/\d+/g); - if (numbers) { - const replies = parseInt(numbers[0]); - const reposts = parseInt(numbers[1]); - const likes = parseInt(numbers[2]); - const bookmarks = parseInt(numbers[3]); - const views = parseInt(numbers[4]); - const newMetrics = { likes, reposts, replies, bookmarks, views }; - log("Metrics extracted from aria-label:", newMetrics); - updateCounter += 1; - // send the metrics if any metric has changed from zero - if ( - (replies > 0 && metrics.replies === 0) || - (reposts > 0 && metrics.reposts === 0) || - (likes > 0 && metrics.likes === 0) || - (bookmarks > 0 && metrics.bookmarks === 0) || - (views > 0 && metrics.views === 0) - ) { - log("Sending first metrics to reportURL:", metrics); - metrics = newMetrics; - sendMetrics(); - updateCounter = 0; - } + const regex = /(\d+)\s+(\w+)/g; + let match; + let newMetrics = { + likes: 0, + reposts: 0, + replies: 0, + bookmarks: 0, + views: 0 + }; + // Iterate over all matches in the aria-label + while ((match = regex.exec(ariaLabel)) !== null) { + const value = parseInt(match[1]); + const name = match[2].toLowerCase(); - // send the metrics if the counter reaches the updateLimit - if (updateCounter >= updateLimit) { - log("Sending metrics to reportURL:", metrics); - metrics = newMetrics; - sendMetrics(); - updateCounter = 0; + if (Object.hasOwnProperty.call(newMetrics, name)) { + newMetrics[name] = value; } } + + log("Metrics extracted from aria-label:", newMetrics); + updateCounter += 1; + // send the metrics if any metric has changed from zero + if ( + (newMetrics.replies > 0 && metrics.replies === 0) || + (newMetrics.reposts > 0 && metrics.reposts === 0) || + (newMetrics.likes > 0 && metrics.likes === 0) || + (newMetrics.bookmarks > 0 && metrics.bookmarks === 0) || + (newMetrics.views > 0 && metrics.views === 0) + ) { + log("Sending first metrics to reportURL:", metrics); + metrics = newMetrics; + sendMetrics(); + updateCounter = 0; + } + + // send the metrics if the counter reaches the updateLimit + if (updateCounter >= updateLimit) { + log("Sending metrics to reportURL:", metrics); + metrics = newMetrics; + sendMetrics(); + updateCounter = 0; + } } function startObservingMetrics(targetNode) { diff --git a/config/routes.rb b/config/routes.rb index cc5ecdf..d89bda0 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,6 @@ Rails.application.routes.draw do devise_for :users - resources :tweets, only: %i[index show] + resources :tweets, only: %i[index show new create] get "track/:id.user.js", to: "tweets#track", as: :track_user_js post "receive_metrics", to: "tweets#receive_metrics", as: :receive_metrics diff --git a/db/migrate/20231008184323_add_avatar_to_tweets.rb b/db/migrate/20231008184323_add_avatar_to_tweets.rb new file mode 100644 index 0000000..ecd28ca --- /dev/null +++ b/db/migrate/20231008184323_add_avatar_to_tweets.rb @@ -0,0 +1,5 @@ +class AddAvatarToTweets < ActiveRecord::Migration[7.1] + def change + add_column :tweets, :avatar, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index b6119e6..0733769 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2023_10_08_101224) do +ActiveRecord::Schema[7.1].define(version: 2023_10_08_184323) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -39,6 +39,7 @@ t.datetime "updated_at", null: false t.bigint "user_id", null: false t.uuid "uuid", default: -> { "gen_random_uuid()" }, null: false + t.string "avatar" t.index ["uuid"], name: "index_tweets_on_uuid", unique: true end diff --git a/spec/models/tweet_spec.rb b/spec/models/tweet_spec.rb index 0a88ecd..9e445d0 100644 --- a/spec/models/tweet_spec.rb +++ b/spec/models/tweet_spec.rb @@ -47,7 +47,7 @@ it "validates presence of url" do new_tweet = Tweet.new(url: nil) expect(new_tweet).to_not be_valid - expect(new_tweet.errors.messages[:url]).to eq(["can't be blank"]) + expect(new_tweet.errors.messages[:url]).to eq(["can't be blank", "is invalid"]) end it "validates uniqueness of uuid" do diff --git a/spec/requests/tweets_spec.rb b/spec/requests/tweets_spec.rb index 945de68..df8f7de 100644 --- a/spec/requests/tweets_spec.rb +++ b/spec/requests/tweets_spec.rb @@ -15,6 +15,20 @@ subject expect(response.body).to include(tweet.author) end + + context "with a tweet with no body" do + let!(:tweet) { create(:tweet, body: nil) } + + it "returns http success" do + subject + expect(response).to have_http_status(:success) + end + + it "returns a list of tweets" do + subject + expect(response.body).to include(tweet.author) + end + end end describe "GET show" do @@ -111,9 +125,9 @@ end context "for a non-logged in user" do - it "returns http not found" do + it "returns http 401" do subject - expect(response).to have_http_status(:not_found) + expect(response).to have_http_status(:unauthorized) end end end From e4c935091728520f754c7462abe76c6b2af3f006 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 23:23:20 +0300 Subject: [PATCH 09/12] Disable tracker debugging --- app/views/tweets/track.js.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/tweets/track.js.erb b/app/views/tweets/track.js.erb index fd17dc1..ac2bae3 100644 --- a/app/views/tweets/track.js.erb +++ b/app/views/tweets/track.js.erb @@ -12,7 +12,7 @@ (function ($) { "use strict"; - const DEBUG = true; // Set to false to disable logging + const DEBUG = false; // Set to false to disable logging const reportURL = "<%= @report_url %>" const uuid = "<%= @tweet.uuid %>" const updateLimit = 5; // Number of updates before sending to metrics to the reportURL From 4acaf545913c822ecb997d35d08849d0421192c1 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 23:24:37 +0300 Subject: [PATCH 10/12] Remove custom logger --- app/views/tweets/track.js.erb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/views/tweets/track.js.erb b/app/views/tweets/track.js.erb index ac2bae3..94da436 100644 --- a/app/views/tweets/track.js.erb +++ b/app/views/tweets/track.js.erb @@ -50,7 +50,6 @@ } function sendMetrics() { - console.log('Test avatar:', getAvatar()); log("Sending metrics to reportURL:", metrics); let data = JSON.stringify({ tweet: { uuid: uuid }, metrics }); if (tweetDataRequired) { From 3db2e4f7f8bc9c5399b7c66e17af5a07bb1e1845 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 23:38:17 +0300 Subject: [PATCH 11/12] Only top 10 tweets with charts --- app/controllers/tweets_controller.rb | 8 ++++++-- app/views/tweets/index.html.slim | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/app/controllers/tweets_controller.rb b/app/controllers/tweets_controller.rb index 6b589a8..71266c6 100644 --- a/app/controllers/tweets_controller.rb +++ b/app/controllers/tweets_controller.rb @@ -4,8 +4,12 @@ class TweetsController < ApplicationController before_action :authenticate_user!, only: %i[new create track] def index - @tweets = Tweet.first(10).map(&:decorate) - + scope = Tweet.joins(:tweet_metrics) + .select("tweets.*, COUNT(tweet_metrics.id) AS metrics_count") + .group("tweets.id") + .order("metrics_count DESC") + @tweets = scope.limit(10).map(&:decorate) + @other_tweets = scope.offset(10).limit(200).map(&:decorate) respond_to do |format| format.html # renders index.html.slim end diff --git a/app/views/tweets/index.html.slim b/app/views/tweets/index.html.slim index 2cbd065..f197a4d 100644 --- a/app/views/tweets/index.html.slim +++ b/app/views/tweets/index.html.slim @@ -5,6 +5,7 @@ section.hero.is-info .columns.is-centered .column.is-1 h1.rotated.has-text-grey Tracked Tweets + / Display top tweets with charts .column.is-10 .section = link_to 'Add Tweet', new_tweet_path, class: 'button is-primary is-fullwidth is-large' @@ -32,3 +33,29 @@ section.hero.is-info span.icon i class="fa fa-twitter" .column.is-1 + / Display other tweets as cards + .columns.is-centered + .column.is-10 + .columns.is-centered + - @other_tweets.each do |tweet| + .column.is-3 + .card + .card-content + .media + .media-left + figure class="image is-48x48" + img src="#{tweet.author_avatar_url}" class="is-rounded" + .media-content + .title.is-4 + = link_to tweet.author_name, tweet.url, target: "_blank" + .content + = tweet.body&.truncate(280) + .card-footer + = link_to tweet_path(tweet), class: "card-footer-item primary" do + span View Metrics + span.icon + i class="fa fa-chart-line" + = link_to tweet.url, class: "card-footer-item", target: "_blank" do + span View Tweet + span.icon + i class="fa fa-twitter" From a2a9998fca31f0fcc7a1c7a4048176998f33a24e Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 8 Oct 2023 23:53:26 +0300 Subject: [PATCH 12/12] Better index layout --- app/controllers/tweets_controller.rb | 2 +- app/decorators/tweet_decorator.rb | 2 +- app/views/tweets/index.html.slim | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/tweets_controller.rb b/app/controllers/tweets_controller.rb index 71266c6..d3fa221 100644 --- a/app/controllers/tweets_controller.rb +++ b/app/controllers/tweets_controller.rb @@ -9,7 +9,7 @@ def index .group("tweets.id") .order("metrics_count DESC") @tweets = scope.limit(10).map(&:decorate) - @other_tweets = scope.offset(10).limit(200).map(&:decorate) + @other_tweets = Tweet.limit(400).map(&:decorate) respond_to do |format| format.html # renders index.html.slim end diff --git a/app/decorators/tweet_decorator.rb b/app/decorators/tweet_decorator.rb index 7e435af..fc5afed 100644 --- a/app/decorators/tweet_decorator.rb +++ b/app/decorators/tweet_decorator.rb @@ -109,7 +109,7 @@ def combined_metrics_series likes_metric(resolution), replies_metric(resolution), reposts_metric(resolution), - bookmarks_metric(resolution), + bookmarks_metric(resolution) # views_metric TODO: enable after getting rid of chartkick and switching to custom Y axis ] end diff --git a/app/views/tweets/index.html.slim b/app/views/tweets/index.html.slim index f197a4d..2c186d3 100644 --- a/app/views/tweets/index.html.slim +++ b/app/views/tweets/index.html.slim @@ -36,11 +36,11 @@ section.hero.is-info / Display other tweets as cards .columns.is-centered .column.is-10 - .columns.is-centered + .columns.is-centered.is-multiline - @other_tweets.each do |tweet| - .column.is-3 - .card - .card-content + .column.is-3.is-flex.is-flex-direction-column + .card.is-flex.is-flex-direction-column.is-flex-grow-1 + .card-content.is-flex.is-flex-direction-column.is-flex-grow-1 .media .media-left figure class="image is-48x48"