Skip to content

Commit

Permalink
Merge branch 'release/2.15.4'
Browse files Browse the repository at this point in the history
  • Loading branch information
vmalloc committed Jan 17, 2019
2 parents de4ea9f + 6e5305f commit 989ce12
Show file tree
Hide file tree
Showing 16 changed files with 111 additions and 34 deletions.
4 changes: 4 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# Changelog


## Version 2.15.4

* Several fixes to Elasticsearch replication logic

## Version 2.15.0

* UPGRADE NOTE: This version changes the docker-compose.yml used to deploy Backslash. Please update
Expand Down
2 changes: 1 addition & 1 deletion _lib/frontend_tmux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ windows:
- pipenv run wsgi
- window_name: celery worker
panes:
- .env/bin/celery -A flask_app.tasks.main worker --loglevel=info -B --max-tasks-per-child=500
- pipenv run celery -A flask_app.tasks.main worker --loglevel=info -B --max-tasks-per-child=500
1 change: 1 addition & 0 deletions api-server/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[package]
name = "api-server"
edition = "2018"
version = "0.1.1"
authors = ["Rotem Yaari"]

Expand Down
1 change: 1 addition & 0 deletions api-server/rustfmt.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
edition = 2018
18 changes: 2 additions & 16 deletions api-server/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,29 +1,15 @@
#![deny(warnings)]
extern crate actix;
extern crate actix_web;
extern crate env_logger;
extern crate failure;
extern crate futures;
extern crate log;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate sentry;
extern crate sentry_actix;
extern crate structopt;
extern crate url;

mod aggregators;
mod state;
mod stats;
mod utils;

use crate::state::AppState;
use crate::stats::StatsCollector;
use actix::prelude::*;
use actix_web::{server, App};
use env_logger::Builder;
use log::info;
use state::AppState;
use stats::StatsCollector;
use std::env;
use std::net::IpAddr;
use structopt::StructOpt;
Expand Down
2 changes: 1 addition & 1 deletion api-server/src/state.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use actix::prelude::*;
use stats::StatsCollector;
use crate::stats::StatsCollector;

pub struct AppState {
pub(crate) stats_collector: Addr<StatsCollector>,
Expand Down
7 changes: 4 additions & 3 deletions api-server/src/stats.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
use actix::prelude::*;
use actix_web::{AsyncResponder, HttpRequest, Query, Responder, State};
use aggregators::{CountHistorgram, DurationAggregator};
use crate::aggregators::{CountHistorgram, DurationAggregator};
use failure::Error;
use state::AppState;
use crate::state::AppState;
use std::collections::HashMap;
use std::fmt::Write;
use std::net::IpAddr;
use std::time::Duration;
use utils::duration_from_secs;
use crate::utils::duration_from_secs;
use serde_derive::Deserialize;

const HISTOGRAM_RESOLUTION_SECONDS: usize = 60;
const HISTOGRAM_NUM_BINS: usize = 10;
Expand Down
1 change: 1 addition & 0 deletions flask_app/blueprints/rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -429,6 +429,7 @@ class ReplicationsResource(ModelResource):
'paused',
'avg_per_second',
'backlog_remaining',
'last_replicated_timestamp',
'last_error',
'service_type',
'username',
Expand Down
11 changes: 9 additions & 2 deletions flask_app/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,7 @@ def last_comment(self):
Index('ix_test_test_info_id_start_time', test_info_id, start_time.desc()),
Index('ix_test_timespan', 'timespan', postgresql_using='gist'),
Index('ix_test_updated_at', updated_at.asc(), postgresql_where=(updated_at != None)),
Index('ix_test_updated_at_id', updated_at.asc(), id.asc()),
)

@rendered_field
Expand Down Expand Up @@ -744,10 +745,16 @@ class Replication(db.Model, TypenameMixin):

_client = None

def __repr__(self):
return f'<Replica to {self.url}>'

def reset(self):
self.paused = True
self.last_chunk_finished = self.last_replicated_id = self.backlog_remaining = \
self.last_error = None
self.last_replicated_timestamp = None
self.last_replicated_id = None
self.last_chunk_finished = None
self.backlog_remaining = None
self.last_error = None
self.untimed_done = False
self.avg_per_second = 0

Expand Down
2 changes: 1 addition & 1 deletion flask_app/tasks/maintenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def reliable_task(task_func, stale_timeout=10 * 60):

@functools.wraps(task_func)
def new_func(*args, **kwargs):
get_redis_client().setex(task_key, 'true', time=stale_timeout)
get_redis_client().setex(name=task_key, value='true', time=stale_timeout)
return task_func(*args, **kwargs)

returned = queue.task(new_func)
Expand Down
26 changes: 26 additions & 0 deletions migrations/versions/8f864a7768f8_add_test_updated_at_id_index.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
"""Add test updated_at,id index
Revision ID: 8f864a7768f8
Revises: 2799ab20163b
Create Date: 2019-01-17 13:13:59.216981
"""

# revision identifiers, used by Alembic.
revision = '8f864a7768f8'
down_revision = '2799ab20163b'

from alembic import op
import sqlalchemy as sa


def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_index('ix_test_updated_at_id', 'test', [sa.text('updated_at ASC'), sa.text('id ASC')], unique=False)
# ### end Alembic commands ###


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index('ix_test_updated_at_id', table_name='test')
# ### end Alembic commands ###
43 changes: 43 additions & 0 deletions scripts/replication.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import time
from contextlib import contextmanager
import click
from pathlib import Path
import sys

sys.path.insert(0, "")

from flask_app.tasks import replications
from flask_app.app import create_app
from flask_app import models


@click.group()
def cli():
pass


@cli.command()
def status():
app = create_app()
with app.app_context():
replica = models.Replication.query.first()
print("Found replica", replica)
with _timing(f"Fetching tests to replicate..."):
tests = list(
models.db.session.execute(
replications._get_tests_to_replicate_query(replica)
)
)
print(f"Found {len(tests)} to replicate. First one is {tests[0]}")


@contextmanager
def _timing(msg):
print(msg, "...")
start_time = time.time()
yield
print(f"... took {time.time() - start_time}s")


if __name__ == "__main__":
cli()
6 changes: 1 addition & 5 deletions webapp/app/admin/replications/index/template.hbs
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,7 @@
</div>
<div class="row">
<div class="col-sm-9 faint">
{{#if replication.backlog_remaining}}Initializing: {{replication.backlog_remaining}}
{{#if replication.active}}
({{replication.time_remaining}} remaining @ {{trunc replication.avg_per_second}} tests/second)
{{/if}}
{{/if}}
Last test replicated from: <span title="{{replication.last_replicated_timestamp_str}}">{{moment-time ago=replication.last_replicated_timestamp}}</span>
</div>
</div>
</div>
Expand Down
2 changes: 1 addition & 1 deletion webapp/app/admin/replications/new/template.hbs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@
<div class="form-group">
{{input type="password" class="form-control" value=model.password placeholder="Password (Optional)"}}
</div>
<input type="submit" class="btn btn-success" {{action "save"}}>
<input type="submit" class="btn btn-success" {{action "save"}} value="Create">
</form>
</div>
8 changes: 4 additions & 4 deletions webapp/app/initializers/moment.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,20 @@ export function initialize(/* application */) {
L: "DD/MM/YYYY",
LL: "D MMMM YYYY",
LLL: "D MMMM YYYY LT",
LLLL: "dddd D MMMM YYYY LT"
LLLL: "dddd D MMMM YYYY LT",
},
calendar: {
lastDay: "[Yesterday at] LTS",
sameDay: "[Today at] LTS",
nextDay: "[Tomorrow at] LTS",
lastWeek: "[last] dddd [at] LTS",
nextWeek: "dddd [at] LTS",
sameElse: "L LTS"
}
sameElse: "L LTS",
},
});
}

export default {
name: "moment",
initialize: initialize
initialize: initialize,
};
11 changes: 11 additions & 0 deletions webapp/app/models/replication.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,17 @@ export default DS.Model.extend({
backlog_remaining: DS.attr("number"),
last_error: DS.attr(),
paused: DS.attr(),
last_replicated_timestamp: DS.attr(),

last_replicated_timestamp_str: computed(
"last_replicated_timestamp",
function() {
let timestamp = this.get("last_replicated_timestamp");
if (timestamp) {
return moment.unix(timestamp);
}
}
),

time_remaining: computed("avg_per_second", "backlog_remaining", function() {
let remaining = this.get("backlog_remaining");
Expand Down

0 comments on commit 989ce12

Please sign in to comment.