From 6fd7bae0ee3f1502709d052c4c54975c2fcf51f3 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Tue, 8 Jan 2019 09:05:22 +0200 Subject: [PATCH 01/10] Convert api-server to Rust 2018 --- api-server/Cargo.toml | 1 + api-server/rustfmt.toml | 1 + api-server/src/main.rs | 18 ++---------------- api-server/src/state.rs | 2 +- api-server/src/stats.rs | 7 ++++--- 5 files changed, 9 insertions(+), 20 deletions(-) create mode 100644 api-server/rustfmt.toml diff --git a/api-server/Cargo.toml b/api-server/Cargo.toml index 88a52ede..84bdb085 100644 --- a/api-server/Cargo.toml +++ b/api-server/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "api-server" +edition = "2018" version = "0.1.1" authors = ["Rotem Yaari"] diff --git a/api-server/rustfmt.toml b/api-server/rustfmt.toml new file mode 100644 index 00000000..a6a8dd3b --- /dev/null +++ b/api-server/rustfmt.toml @@ -0,0 +1 @@ +edition = 2018 diff --git a/api-server/src/main.rs b/api-server/src/main.rs index c9969ccc..6dfe2a40 100644 --- a/api-server/src/main.rs +++ b/api-server/src/main.rs @@ -1,29 +1,15 @@ #![deny(warnings)] -extern crate actix; -extern crate actix_web; -extern crate env_logger; -extern crate failure; -extern crate futures; -extern crate log; -extern crate serde; -#[macro_use] -extern crate serde_derive; -extern crate sentry; -extern crate sentry_actix; -extern crate structopt; -extern crate url; - mod aggregators; mod state; mod stats; mod utils; +use crate::state::AppState; +use crate::stats::StatsCollector; use actix::prelude::*; use actix_web::{server, App}; use env_logger::Builder; use log::info; -use state::AppState; -use stats::StatsCollector; use std::env; use std::net::IpAddr; use structopt::StructOpt; diff --git a/api-server/src/state.rs b/api-server/src/state.rs index e92809ec..0f7f68f6 100644 --- a/api-server/src/state.rs +++ b/api-server/src/state.rs @@ -1,5 +1,5 @@ use actix::prelude::*; -use stats::StatsCollector; +use crate::stats::StatsCollector; pub struct AppState { pub(crate) stats_collector: Addr, diff --git a/api-server/src/stats.rs b/api-server/src/stats.rs index 42d1c8dc..5b63c818 100644 --- a/api-server/src/stats.rs +++ b/api-server/src/stats.rs @@ -1,13 +1,14 @@ use actix::prelude::*; use actix_web::{AsyncResponder, HttpRequest, Query, Responder, State}; -use aggregators::{CountHistorgram, DurationAggregator}; +use crate::aggregators::{CountHistorgram, DurationAggregator}; use failure::Error; -use state::AppState; +use crate::state::AppState; use std::collections::HashMap; use std::fmt::Write; use std::net::IpAddr; use std::time::Duration; -use utils::duration_from_secs; +use crate::utils::duration_from_secs; +use serde_derive::Deserialize; const HISTOGRAM_RESOLUTION_SECONDS: usize = 60; const HISTOGRAM_NUM_BINS: usize = 10; From 3fa750de9ddd39e571f10f5af2333a57145e5c8a Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Thu, 10 Jan 2019 14:21:02 +0200 Subject: [PATCH 02/10] Fix replica reset operation (closes #470) --- flask_app/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flask_app/models.py b/flask_app/models.py index 21ef64b8..2f1db105 100644 --- a/flask_app/models.py +++ b/flask_app/models.py @@ -746,8 +746,11 @@ class Replication(db.Model, TypenameMixin): def reset(self): self.paused = True - self.last_chunk_finished = self.last_replicated_id = self.backlog_remaining = \ - self.last_error = None + self.last_replicated_timestamp = None + self.last_replicated_id = None + self.last_chunk_finished = None + self.backlog_remaining = None + self.last_error = None self.untimed_done = False self.avg_per_second = 0 From 5fec2de85de39471fcdc5055ceac29cf9953b932 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Mon, 14 Jan 2019 10:53:44 +0200 Subject: [PATCH 03/10] Add script to inspect status of replica objects --- flask_app/models.py | 3 +++ scripts/replication.py | 45 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 scripts/replication.py diff --git a/flask_app/models.py b/flask_app/models.py index 2f1db105..bba4de8d 100644 --- a/flask_app/models.py +++ b/flask_app/models.py @@ -744,6 +744,9 @@ class Replication(db.Model, TypenameMixin): _client = None + def __repr__(self): + return f'' + def reset(self): self.paused = True self.last_replicated_timestamp = None diff --git a/scripts/replication.py b/scripts/replication.py new file mode 100644 index 00000000..55eb396a --- /dev/null +++ b/scripts/replication.py @@ -0,0 +1,45 @@ +import time +from contextlib import contextmanager +import click +from pathlib import Path +import sys + +sys.path.insert(0, "") + +from flask_app.tasks import replications +from flask_app.app import create_app +from flask_app import models + + +@click.group() +def cli(): + pass + + +@cli.command() +def status(): + num_iterations = 3 + app = create_app() + with app.app_context(): + replica = models.Replication.query.first() + print("Found replica", replica) + for i in range(num_iterations): + with _timing(f"Fetching tests to replicate #{i+1}/{num_iterations}"): + tests = list( + models.db.session.execute( + replications._get_tests_to_replicate_query(replica) + ) + ) + print(f"Found {len(tests)} to replicate. First one is {tests[0]}") + + +@contextmanager +def _timing(msg): + print(msg, "...") + start_time = time.time() + yield + print(f"... took {time.time() - start_time}s") + + +if __name__ == "__main__": + cli() From 622beaaa0f72c94a15b3ee6dea5c4f5f3535cbe2 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Wed, 16 Jan 2019 14:31:53 +0200 Subject: [PATCH 04/10] Fix replication script --- scripts/replication.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/replication.py b/scripts/replication.py index 55eb396a..45c75c88 100644 --- a/scripts/replication.py +++ b/scripts/replication.py @@ -23,14 +23,13 @@ def status(): with app.app_context(): replica = models.Replication.query.first() print("Found replica", replica) - for i in range(num_iterations): - with _timing(f"Fetching tests to replicate #{i+1}/{num_iterations}"): - tests = list( - models.db.session.execute( - replications._get_tests_to_replicate_query(replica) - ) + with _timing(f"Fetching tests to replicate..."): + tests = list( + models.db.session.execute( + replications._get_tests_to_replicate_query(replica, bulk_size=5) ) - print(f"Found {len(tests)} to replicate. First one is {tests[0]}") + ) + print(f"Found {len(tests)} to replicate. First one is {tests[0]}") @contextmanager From 6f68a9501f7601bcb56f88804e14fda28f507ba0 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Thu, 17 Jan 2019 13:06:13 +0200 Subject: [PATCH 05/10] Fix replication test script --- scripts/replication.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/replication.py b/scripts/replication.py index 45c75c88..fcc55ef6 100644 --- a/scripts/replication.py +++ b/scripts/replication.py @@ -18,7 +18,6 @@ def cli(): @cli.command() def status(): - num_iterations = 3 app = create_app() with app.app_context(): replica = models.Replication.query.first() @@ -26,7 +25,7 @@ def status(): with _timing(f"Fetching tests to replicate..."): tests = list( models.db.session.execute( - replications._get_tests_to_replicate_query(replica, bulk_size=5) + replications._get_tests_to_replicate_query(replica) ) ) print(f"Found {len(tests)} to replicate. First one is {tests[0]}") From 7339a0ad6a2c350e750ff79b3176d7ca0bfe0e1f Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Thu, 17 Jan 2019 13:11:15 +0200 Subject: [PATCH 06/10] Expose last replicated timestamp (closes #474) --- flask_app/blueprints/rest.py | 1 + webapp/app/admin/replications/index/template.hbs | 6 +----- webapp/app/admin/replications/new/template.hbs | 2 +- webapp/app/initializers/moment.js | 8 ++++---- webapp/app/models/replication.js | 11 +++++++++++ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/flask_app/blueprints/rest.py b/flask_app/blueprints/rest.py index fcb402c2..3ef2c902 100644 --- a/flask_app/blueprints/rest.py +++ b/flask_app/blueprints/rest.py @@ -429,6 +429,7 @@ class ReplicationsResource(ModelResource): 'paused', 'avg_per_second', 'backlog_remaining', + 'last_replicated_timestamp', 'last_error', 'service_type', 'username', diff --git a/webapp/app/admin/replications/index/template.hbs b/webapp/app/admin/replications/index/template.hbs index 0f4493cf..89cdf276 100644 --- a/webapp/app/admin/replications/index/template.hbs +++ b/webapp/app/admin/replications/index/template.hbs @@ -31,11 +31,7 @@
- {{#if replication.backlog_remaining}}Initializing: {{replication.backlog_remaining}} - {{#if replication.active}} - ({{replication.time_remaining}} remaining @ {{trunc replication.avg_per_second}} tests/second) - {{/if}} - {{/if}} + Last test replicated from: {{moment-time ago=replication.last_replicated_timestamp}}
diff --git a/webapp/app/admin/replications/new/template.hbs b/webapp/app/admin/replications/new/template.hbs index c3cfec17..2af06ae5 100644 --- a/webapp/app/admin/replications/new/template.hbs +++ b/webapp/app/admin/replications/new/template.hbs @@ -15,6 +15,6 @@
{{input type="password" class="form-control" value=model.password placeholder="Password (Optional)"}}
- + diff --git a/webapp/app/initializers/moment.js b/webapp/app/initializers/moment.js index 5a8f997c..335d07d6 100644 --- a/webapp/app/initializers/moment.js +++ b/webapp/app/initializers/moment.js @@ -8,7 +8,7 @@ export function initialize(/* application */) { L: "DD/MM/YYYY", LL: "D MMMM YYYY", LLL: "D MMMM YYYY LT", - LLLL: "dddd D MMMM YYYY LT" + LLLL: "dddd D MMMM YYYY LT", }, calendar: { lastDay: "[Yesterday at] LTS", @@ -16,12 +16,12 @@ export function initialize(/* application */) { nextDay: "[Tomorrow at] LTS", lastWeek: "[last] dddd [at] LTS", nextWeek: "dddd [at] LTS", - sameElse: "L LTS" - } + sameElse: "L LTS", + }, }); } export default { name: "moment", - initialize: initialize + initialize: initialize, }; diff --git a/webapp/app/models/replication.js b/webapp/app/models/replication.js index 0c79b595..d7a2c706 100644 --- a/webapp/app/models/replication.js +++ b/webapp/app/models/replication.js @@ -11,6 +11,17 @@ export default DS.Model.extend({ backlog_remaining: DS.attr("number"), last_error: DS.attr(), paused: DS.attr(), + last_replicated_timestamp: DS.attr(), + + last_replicated_timestamp_str: computed( + "last_replicated_timestamp", + function() { + let timestamp = this.get("last_replicated_timestamp"); + if (timestamp) { + return moment.unix(timestamp); + } + } + ), time_remaining: computed("avg_per_second", "backlog_remaining", function() { let remaining = this.get("backlog_remaining"); From 5060b93887eb0d8bc3a39d99fce83c67ec99a457 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Thu, 17 Jan 2019 13:16:45 +0200 Subject: [PATCH 07/10] Add index to tests for replication optimization (closes #472) --- flask_app/models.py | 1 + ...864a7768f8_add_test_updated_at_id_index.py | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 migrations/versions/8f864a7768f8_add_test_updated_at_id_index.py diff --git a/flask_app/models.py b/flask_app/models.py index bba4de8d..8ca1982d 100644 --- a/flask_app/models.py +++ b/flask_app/models.py @@ -476,6 +476,7 @@ def last_comment(self): Index('ix_test_test_info_id_start_time', test_info_id, start_time.desc()), Index('ix_test_timespan', 'timespan', postgresql_using='gist'), Index('ix_test_updated_at', updated_at.asc(), postgresql_where=(updated_at != None)), + Index('ix_test_updated_at_id', updated_at.asc(), id.asc()), ) @rendered_field diff --git a/migrations/versions/8f864a7768f8_add_test_updated_at_id_index.py b/migrations/versions/8f864a7768f8_add_test_updated_at_id_index.py new file mode 100644 index 00000000..25b3ffc8 --- /dev/null +++ b/migrations/versions/8f864a7768f8_add_test_updated_at_id_index.py @@ -0,0 +1,26 @@ +"""Add test updated_at,id index + +Revision ID: 8f864a7768f8 +Revises: 2799ab20163b +Create Date: 2019-01-17 13:13:59.216981 + +""" + +# revision identifiers, used by Alembic. +revision = '8f864a7768f8' +down_revision = '2799ab20163b' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index('ix_test_updated_at_id', 'test', [sa.text('updated_at ASC'), sa.text('id ASC')], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_test_updated_at_id', table_name='test') + # ### end Alembic commands ### From 60dbcfae7a8330141c27969f69b791e0350a231f Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Thu, 17 Jan 2019 13:28:04 +0200 Subject: [PATCH 08/10] Fix call to Redis' setex() --- flask_app/tasks/maintenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flask_app/tasks/maintenance.py b/flask_app/tasks/maintenance.py index 554b43db..081127d1 100644 --- a/flask_app/tasks/maintenance.py +++ b/flask_app/tasks/maintenance.py @@ -23,7 +23,7 @@ def reliable_task(task_func, stale_timeout=10 * 60): @functools.wraps(task_func) def new_func(*args, **kwargs): - get_redis_client().setex(task_key, 'true', time=stale_timeout) + get_redis_client().setex(name=task_key, value='true', time=stale_timeout) return task_func(*args, **kwargs) returned = queue.task(new_func) From 67d1a84b09e34dd878633c0d01f201a7d4340354 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Thu, 17 Jan 2019 13:28:43 +0200 Subject: [PATCH 09/10] Fix command-line running of celery in tmux --- _lib/frontend_tmux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_lib/frontend_tmux.yml b/_lib/frontend_tmux.yml index 8540fbc7..df3411c5 100644 --- a/_lib/frontend_tmux.yml +++ b/_lib/frontend_tmux.yml @@ -7,4 +7,4 @@ windows: - pipenv run wsgi - window_name: celery worker panes: - - .env/bin/celery -A flask_app.tasks.main worker --loglevel=info -B --max-tasks-per-child=500 + - pipenv run celery -A flask_app.tasks.main worker --loglevel=info -B --max-tasks-per-child=500 From 6e5305fa51ec8671c895bf15b9ad84a90a6302a3 Mon Sep 17 00:00:00 2001 From: Rotem Yaari Date: Thu, 17 Jan 2019 14:01:59 +0200 Subject: [PATCH 10/10] Changelog --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a75b4593..ef9e8ff3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ # Changelog +## Version 2.15.4 + +* Several fixes to Elasticsearch replication logic + ## Version 2.15.0 * UPGRADE NOTE: This version changes the docker-compose.yml used to deploy Backslash. Please update