diff --git a/buckaroo/all_transforms.py b/buckaroo/all_transforms.py index eff7a4dc..9b4613c5 100644 --- a/buckaroo/all_transforms.py +++ b/buckaroo/all_transforms.py @@ -1,8 +1,10 @@ -from .lispy import s -from .configure_utils import configure_buckaroo import pandas as pd import numpy as np +from .lispy import s +from .configure_utils import configure_buckaroo +from .cleaning_commands import (to_bool, to_datetime, to_int, to_float, to_string) + class Command(object): pass @@ -122,7 +124,7 @@ def transform_to_py(df, col, col_spec): -class to_datetime(Command): +class ato_datetime(Command): #argument_names = ["df", "col"] command_default = [s('to_datetime'), s('df'), "col"] command_pattern = [None] @@ -154,6 +156,7 @@ def transform_to_py(df, col): " df.drop('%s', axis=1, inplace=True)" % col, " df.index = old_col.values"]) -DefaultCommandKlsList = [DropCol, to_datetime, SafeInt, FillNA, reindex, OneHot, GroupBy] +DefaultCommandKlsList = [DropCol, SafeInt, FillNA, reindex, OneHot, GroupBy, + to_bool, to_datetime, to_int, to_float, to_string] command_defaults, command_patterns, buckaroo_transform, buckaroo_to_py_core = configure_buckaroo(DefaultCommandKlsList) diff --git a/buckaroo/analysis.py b/buckaroo/analysis.py index 260f6e5b..49350284 100644 --- a/buckaroo/analysis.py +++ b/buckaroo/analysis.py @@ -91,18 +91,107 @@ def summary(sampled_ser, summary_ser, ser): def int_digits(n): + if np.isnan(n): + return 1 if n == 0: return 1 if np.sign(n) == -1: return int(np.floor(np.log10(np.abs(n)))) + 2 return int(np.floor(np.log10(n)+1)) -def histogram(ser): - raw_counts, bins = np.histogram(ser, 10) - scaled_counts = np.round(raw_counts/raw_counts.sum(),2) - return [scaled_counts, bins] +def numeric_histogram_labels(endpoints): + left = endpoints[0] + labels = [] + for edge in endpoints[1:]: + labels.append("{:.0f}-{:.0f}".format(left, edge)) + left = edge + return labels +#histogram_labels(endpoints) + +def numeric_histogram(arr, nan_per): + ret_histo = [] + nan_observation = {'name':'NA', 'NA':np.round(nan_per*100, 0)} + if nan_per == 1.0: + return [nan_observation] + + vals = arr.dropna() + low_tail, high_tail = np.quantile(vals, 0.01), np.quantile(vals, 0.99) + low_pass = arr>low_tail + high_pass = arr < high_tail + meat = vals[low_pass & high_pass] + populations, endpoints =np.histogram(meat, 10) + + labels = numeric_histogram_labels(endpoints) + normalized_pop = populations / populations.sum() + low_label = "%r - %r" % (vals.min(), low_tail) + high_label = "%r - %r" % (high_tail, vals.max()) + ret_histo.append({'name': low_label, 'tail':1}) + for label, pop in zip(labels, normalized_pop): + ret_histo.append({'name': label, 'population':np.round(pop * 100, 0)}) + high_label = "%r - %r" % (high_tail, vals.max()) + ret_histo.append({'name': high_label, 'tail':1}) + if nan_per > 0.0: + ret_histo.append(nan_observation) + return ret_histo + + +def histo_format(v, l): + scaled = v/l + + +def categorical_dict(ser, val_counts, top_n_positions=7): + l = len(ser) + top = min(len(val_counts), top_n_positions) + + + top_vals = val_counts.iloc[:top] + #top_percentage = top_vals.sum() / l + #if len(val_counts) > 5 and top_percentage < .05: + + rest_vals = val_counts.iloc[top:] + histogram = top_vals.to_dict() + + + full_long_tail = rest_vals.sum() + unique_count = sum(val_counts == 1) + long_tail = full_long_tail - unique_count + if unique_count > 0: + histogram['unique'] = np.round( (unique_count/l)* 100, 0) + if long_tail > 0: + histogram['longtail'] = np.round((long_tail/l) * 100,0) + return histogram + +def categorical_histogram(ser, val_counts, nan_per, top_n_positions=7): + nan_observation = {'name':'NA', 'NA':np.round(nan_per*100, 0)} + cd = categorical_dict(ser, val_counts, top_n_positions) + + l = len(ser) + histogram = [] + longtail_obs = {'name': 'longtail'} + for k,v in cd.items(): + if k in ["longtail", "unique"]: + longtail_obs[k] = v + continue + histogram.append({'name':k, 'cat_pop': np.round((v/l)*100,0) }) + if len(longtail_obs) > 1: + histogram.append(longtail_obs) + if nan_per > 0.0: + histogram.append(nan_observation) + return histogram + + +def histogram(ser, nan_per): + is_numeric = pd.api.types.is_numeric_dtype(ser.dtype) + val_counts = ser.value_counts() + if is_numeric and len(val_counts)>5: + temp_histo = numeric_histogram(ser, nan_per) + if len(temp_histo) > 5: + #if we had basically a categorical variable encoded into an integer.. don't return it + return temp_histo + return categorical_histogram(ser, val_counts, nan_per) + class ColDisplayHints(ColAnalysis): requires_summary = ['min', 'max'] # What summary stats does this analysis provide provided_summary = [] @@ -111,15 +200,16 @@ class ColDisplayHints(ColAnalysis): 'is_numeric', 'is_integer', 'min_digits', 'max_digits', 'histogram'] @staticmethod - def col_hints(sampled_ser, summary_ser, ser): - is_numeric = pd.api.types.is_numeric_dtype(ser.dtype) - if not is_numeric: - return dict(is_numeric=False) - if len(ser) == 0: - return dict(is_numeric=False) + def table_hints(sampled_ser, summary_ser, table_hint_col_dict): + is_numeric = pd.api.types.is_numeric_dtype(sampled_ser.dtype) + # if not is_numeric: + # return dict(is_numeric=False) + # if len(sampled_ser) == 0: + # return dict(is_numeric=False) return dict( - is_numeric=True, - is_integer=pd.api.types.is_integer_dtype(ser), - min_digits=int_digits(summary_ser.loc['min']), - max_digits=int_digits(summary_ser.loc['max']), - histogram=histogram(ser)) + is_numeric=is_numeric, + is_integer=pd.api.types.is_integer_dtype(sampled_ser), + min_digits=(is_numeric and int_digits(summary_ser.loc['min'])) or 0, + max_digits=(is_numeric and int_digits(summary_ser.loc['max'])) or 0, + histogram=histogram(sampled_ser, summary_ser['nan_per'])) + diff --git a/buckaroo/analysis_management.py b/buckaroo/analysis_management.py index b5eff8ed..e707eb63 100644 --- a/buckaroo/analysis_management.py +++ b/buckaroo/analysis_management.py @@ -1,6 +1,6 @@ import numpy as np import pandas as pd - +import traceback from buckaroo.pluggable_analysis_framework import ( ColAnalysis, order_analysis, check_solvable, NotProvidedException) @@ -28,11 +28,13 @@ def produce_summary_df(df, ordered_objs, df_name='test_df'): summary_res = a_kls.summary(ser, summary_ser, ser) for k,v in summary_res.items(): summary_ser.loc[k] = v - for k,v in a_kls.table_hints(sampled_ser, summary_ser, table_hint_dict): + th_dict = a_kls.table_hints(sampled_ser, summary_ser, table_hint_dict) + for k,v in th_dict.items(): table_hint_dict[k] = v except Exception as e: print("summary_ser", summary_ser) errs[ser_name] = e, a_kls + traceback.print_exc() continue summary_col_dict[ser_name] = summary_ser table_hint_col_dict[ser_name] = table_hint_dict diff --git a/buckaroo/auto_clean.py b/buckaroo/auto_clean.py new file mode 100644 index 00000000..7cfd7f68 --- /dev/null +++ b/buckaroo/auto_clean.py @@ -0,0 +1,208 @@ +import sys +import math +import warnings + +from datetime import timedelta +from collections import defaultdict + +import pandas as pd +import numpy as np + + +#adapted from https://docs.python.org/3/library/warnings.html + +# this is needed to see if a series that we think should be datetime +# would throw a warning indicating a slow coercion this saves a huge +# (100x) slowdown + +def check_for_datetime_warnings(ser): + "return 1 if to_datetime throws errors for the series" + with warnings.catch_warnings(record=True) as w: + pd.to_datetime(ser, errors='coerce') + if len(w) == 0: + return 0 + if "Could not infer format" in str(w[-1].message): + return 1 + else: + #not sure how we'd get here + return 1 + +default_type_dict = { + 'datetime':0, 'datetime_error':0, + 'int':0, 'int_error':0, + 'float':0, 'float_error':0, + 'bool':0, 'bool_error':0} + + +#pandas parses ints and floats as datetimes in ns, you end up with a +# lot of values around 1970 Unix epoch we use 1971 as a cutoff, a bit +# of a hack, but pragmatic +SEVENTY_ONE_CUTOFF = pd.to_datetime('1971-01-01') + +def get_object_typing_metadata(ser): + # this function should just return percentages, to separate + # introspection from action. This way we can pass in a different + # decision weighting. As is this function is complex enough + counts = defaultdict(lambda: 0) + counts.update(default_type_dict) # we always want these keys present + assert pd.api.types.is_object_dtype(ser.dtype) + #this is slow because it goes through python as opposed to vectorized C code + for v in ser.values: + try: + dt = pd.to_datetime(v) + if dt > SEVENTY_ONE_CUTOFF: + counts['datetime'] += 1 + else: + counts['datetime_error'] += 1 + except (pd.core.tools.datetimes.DateParseError, ValueError, TypeError): + counts['datetime_error'] += 1 + try: + int(v) + counts['int'] += 1 + except ValueError: + counts['int_error'] += 1 + try: + float(v) + counts['float'] += 1 + except ValueError: + counts['float_error'] += 1 + + if isinstance(v, bool): + counts['bool'] += 1 + else: + counts['bool_error'] += 1 + + if len(ser) == 0: + return counts + ret_dict = dict([[k, v/len(ser)] for k,v in counts.items()]) + + #this is an ugly hack, but it really speeds things up if there are + #abberant kind of datetime columns + if ret_dict['datetime_error'] < .5: + if check_for_datetime_warnings(ser): + ret_dict['datetime_error'] = 1.0 + if ret_dict['int_error'] < .5: + float_remainder = (pd.to_numeric(ser, errors='coerce').abs() % 1).sum() + if float_remainder > 0.0001: + ret_dict['int_error'] = 1 + return ret_dict + +def get_typing_metadata(ser): + td = type_dict = default_type_dict.copy() + dt = ser.dtype + if not pd.api.types.is_object_dtype(dt): + td['exact_type'] = str(dt) + if pd.api.types.is_datetime64_any_dtype(dt): + #general_type is used as a pass through + td['general_type'] = 'datetime' + elif pd.api.types.is_bool_dtype(dt): + td['general_type'] = 'bool' + elif pd.api.types.is_categorical_dtype(dt): + pass #not sure how to handle this yet, it will end up being handled as an object/string + elif pd.api.types.is_float_dtype(dt): + #could still be a float that includes only ints + td['general_type'] = 'float' + td['float'], td['float_error'] = 1, 0 + elif pd.api.types.is_integer_dtype(dt): + td['general_type'] = 'int' + td['int'], td['int_error'] = 1, 0 + return td + else: + return get_object_typing_metadata(ser.dropna()) + +def recommend_type(type_dict): + if type_dict.get('general_type', None) is not None: + return type_dict['general_type'] + if type_dict['datetime_error'] < 0.5: + return 'datetime' + if type_dict['bool_error'] < 0.5: + #bool ends up being stricter than int or float + return 'bool' + if type_dict['float_error'] < 0.5 or type_dict['int_error'] < 0.5: + #numeric type, figure out if float or int float will parse for + # everything that also parses as int + if math.isclose(type_dict['float'], type_dict['int']) or type_dict['int'] > type_dict['float']: + return 'int' + else: + return 'float' + return 'string' + +def smart_to_int(ser): + + if pd.api.types.is_numeric_dtype(ser): + working_ser = ser + lower, upper = ser.min(), ser.max() + else: + working_ser = pd.to_numeric(ser, errors='coerce') + lower, upper = working_ser.min(), working_ser.max() + + + if lower < 0: + if upper < np.iinfo(np.int8).max: + new_type = 'Int8' + elif upper < np.iinfo(np.int16).max: + new_type = 'Int16' + elif upper < np.iinfo(np.int32).max: + new_type = 'Int32' + else: + new_type = 'Int64' + else: + if upper < np.iinfo(np.uint8).max: + new_type = 'UInt8' + elif upper < np.iinfo(np.uint16).max: + new_type = 'UInt16' + elif upper < np.iinfo(np.uint32).max: + new_type = 'UInt32' + else: + new_type = 'UInt64' + base_ser = pd.to_numeric(ser, errors='coerce').dropna() + return base_ser.astype(new_type).reindex(ser.index) + +def coerce_series(ser, new_type): + if new_type == 'bool': + #dropna is key here, otherwise Nan's and errors are treated as true + return pd.to_numeric(ser, errors='coerce').dropna().astype('boolean').reindex(ser.index) + elif new_type == 'datetime': + return pd.to_datetime(ser, errors='coerce').reindex(ser.index) + elif new_type == 'int': + # try: + return smart_to_int(ser) + # except: + # #just let pandas figure it out, we recommended the wrong type + # return pd.to_numeric(ser, errors='coerce') + + elif new_type == 'float': + return pd.to_numeric(ser, errors='coerce').dropna().astype('float').reindex(ser.index) + elif new_type == 'string': + return ser.fillna(value="").astype('string').replace("", None).reindex(ser.index) + else: + raise Exception("Unkown type of %s" % new_type) + +def emit_command(col_name, new_type): + # I need a "no-op" new_type that doesn't change a column at all + # also possible meta tags about commands taht will change data, vs just re-typing + return [{"symbol":"to_%s" % new_type , "meta":{"precleaning":True}},{"symbol":"df"}, col_name] + +def auto_type_df(df): + #this is much faster because we only run the slow function on a maximum of 200 rows. + #That's a good size for an estimate + sample_size = min(len(df), 200) + recommended_types = {} + new_data = {} + for c in df.columns: + recommended_types[c] = recommend_type(get_typing_metadata(df[c].sample(sample_size))) + new_data[c] = coerce_series(df[c], recommended_types[c]) + return pd.DataFrame(new_data) + +def get_auto_type_operations(df): + #this is much faster because we only run the slow function on a maximum of 200 rows. + #That's a good size for an estimate + sample_size = min(len(df), 200) + cleaning_commands = [] + for c in df.columns: + new_type = recommend_type(get_typing_metadata(df[c].sample(sample_size))) + cleaning_commands.append(emit_command(c, new_type)) + return cleaning_commands + + + diff --git a/buckaroo/buckaroo_widget.py b/buckaroo/buckaroo_widget.py index 831bd679..8d42d187 100644 --- a/buckaroo/buckaroo_widget.py +++ b/buckaroo/buckaroo_widget.py @@ -8,17 +8,25 @@ TODO: Add module docstring """ import json +import warnings from ipywidgets import DOMWidget from traitlets import Unicode, List, Dict, observe from ._frontend import module_name, module_version from .all_transforms import configure_buckaroo, DefaultCommandKlsList +from .lisp_utils import (lists_match, split_operations) + +from .auto_clean import get_auto_type_operations from .down_sample import sample from .analysis import (TypingStats, DefaultSummaryStats, ColDisplayHints) + + from .analysis_management import DfStats + + from pandas.io.json import dumps as pdumps @@ -51,6 +59,7 @@ def df_to_obj(df, order = None, table_hints=None): obj['schema'] = dict(fields=fields) return obj + FAST_SUMMARY_WHEN_GREATER = 1_000_000 class BuckarooWidget(DOMWidget): """TODO: Add docstring here @@ -64,13 +73,15 @@ class BuckarooWidget(DOMWidget): commandConfig = Dict({}).tag(sync=True) operations = List().tag(sync=True) - + machine_gen_operations = List().tag(sync=True) command_classes = DefaultCommandKlsList origDf = Dict({}).tag(sync=True) summaryDf = Dict({}).tag(sync=True) - operation_results = Dict({}).tag(sync=True) + operation_results = Dict( + {'transformed_df':EMPTY_DF_OBJ, 'generated_py_code':'# instantiation, unused'} + ).tag(sync=True) dfConfig = Dict( { @@ -85,104 +96,119 @@ class BuckarooWidget(DOMWidget): 'showCommands': True, }).tag(sync=True) - - def __init__(self, df, - sampled=True, - summaryStats=False, - reorderdColumns=False, - showTransformed=True, - showCommands=True, - really_reorder_columns=False): - super().__init__() + def should_sample(self, df, sampled, reorderdColumns): rows = len(df) cols = len(df.columns) item_count = rows * cols - fast_mode = sampled or reorderdColumns if item_count > FAST_SUMMARY_WHEN_GREATER: fast_mode = True - elif really_reorder_columns: #an override - fast_mode = True if fast_mode: - self.dfConfig['sampled'] = True - - self.stats = DfStats(df, [TypingStats, DefaultSummaryStats, ColDisplayHints]) - self.summaryDf = df_to_obj(self.stats.presentation_sdf, self.stats.col_order) + return True + return False + def get_df_config(self, df, sampled, reorderdColumns, showCommands): tempDfc = self.dfConfig.copy() tempDfc.update(dict( totalRows=len(df), columns=len(df.columns), - showTransformed=showTransformed, + #removing showCommands for now, mirroring showTransformed showCommands=showCommands)) + tempDfc['sampled'] = self.should_sample(df, sampled, reorderdColumns) + return tempDfc + + def __init__(self, df, + sampled=True, + summaryStats=False, + reorderdColumns=False, + showCommands=True): + super().__init__() + warnings.filterwarnings('ignore') + #moving setup_from_command_kls_list early in the init because + #it's relatively benign and not tied to other linked updates - self.df = df - self.dfConfig = tempDfc - #just called to trigger setting origDf properly - self.update_based_on_df_config(3) - self.operation_results = { - 'transformed_df':self.origDf, - 'generated_py_code':'#from py widget init'} self.setup_from_command_kls_list() + self.dfConfig = self.get_df_config(df, sampled, reorderdColumns, showCommands) + #we need dfConfig setup first before we get the proper + #working_df and generate the typed_df + self.raw_df = df - def add_analysis(self, analysis_obj): - self.stats.add_analysis(analysis_obj) - self.summaryDf = df_to_obj(self.stats.presentation_sdf, self.stats.col_order) - #just trigger redisplay - self.update_based_on_df_config(3) + # this will trigger the setting of self.typed_df + self.operations = get_auto_type_operations(df) + warnings.filterwarnings('default') @observe('dfConfig') def update_based_on_df_config(self, change): - tdf = self.df_from_dfConfig() - if self.dfConfig['reorderdColumns']: - #ideally this won't require a reserialization. All - #possible col_orders shoudl be serialized once, and the - #frontend should just toggle from them - #self.origDf = df_to_obj(tdf, self.stats.col_order, table_hints=self.stats.table_hints) - self.origDf = df_to_obj(tdf, self.stats.col_order) #, table_hints=self.stats.table_hints) - else: - self.origDf = df_to_obj(tdf) #, table_hints=self.stats.table_hints) - - def df_from_dfConfig(self): - if self.dfConfig['sampled']: - return sample(self.df, self.dfConfig['sampleSize']) - else: - return self.df + if hasattr(self, 'typed_df'): + self.origDf = df_to_obj(self.typed_df, self.typed_df.columns, table_hints=self.stats.table_hints) + #otherwise this is a call before typed_df has been completely setup @observe('operations') - def interpret_operations(self, change): - print("interpret_operations") + def handle_operations(self, change): + if lists_match(change['old'], change['new']): + return + new_ops = change['new'] + split_ops = split_operations(new_ops) + self.machine_gen_operations = split_ops[0] + + user_gen_ops = split_ops[1] + + #if either the user_gen part or the machine_gen part changes, + #we still have to recompute the generated code and + #resulting_df because the input df will be different + results = {} - results['generated_py_code'] = 'before interpreter' try: - operations = [{'symbol': 'begin'}] - operations.extend(change['new']) - #print("interpret_operations", operations) - - if len(operations) == 1: - results['transformed_df'] = self.origDf - results['generated_py_code'] = 'no operations' - #print('exiting early') - return - #generating python code seems slightly less error prone than the transform - results['generated_py_code'] = self.buckaroo_to_py_core(operations[1:]) - #note doesn't use df_to_obj - transformed_df = self.buckaroo_transform(operations, self.df) + transformed_df = self.interpret_ops(user_gen_ops, self.typed_df) + #note we call gneerate_py_code based on the full + #self.operations, this makes sure that machine_gen + #cleaning code shows up too + results['generated_py_code'] = self.generate_code(new_ops) results['transformed_df'] = json.loads(transformed_df.to_json(orient='table', indent=2)) results['transform_error'] = False - except Exception as e: results['transformed_df'] = EMPTY_DF_OBJ print(e) results['transform_error'] = str(e) self.operation_results = results + @observe('machine_gen_operations') + def interpret_machine_gen_ops(self, change, force=False): + if (not force) and lists_match(change['old'], change['new']): + return # nothing changed, do no computations + new_ops = change['new'] + + #this won't listen to sampled changes proeprly + if self.dfConfig['sampled']: + working_df = sample(self.raw_df, self.dfConfig['sampleSize']) + else: + working_df = self.raw_df + self.typed_df = self.interpret_ops(new_ops, working_df) + + # stats need to be rerun each time + self.stats = DfStats(self.typed_df, [TypingStats, DefaultSummaryStats, ColDisplayHints]) + self.summaryDf = df_to_obj(self.stats.presentation_sdf, self.stats.col_order) + self.update_based_on_df_config(3) + + def generate_code(self, operations): + if len(operations) == 0: + return 'no operations' + return self.buckaroo_to_py_core(operations) + + def interpret_ops(self, new_ops, df): + operations = [{'symbol': 'begin'}] + operations.extend(new_ops) + if len(operations) == 1: + return df + return self.buckaroo_transform(operations , df) + def setup_from_command_kls_list(self): + #used to initially setup the interpreter, and when a command + #is added interactively command_defaults, command_patterns, self.buckaroo_transform, self.buckaroo_to_py_core = configure_buckaroo( self.command_classes) - self.commandConfig = dict( - argspecs=command_patterns, defaultArgs=command_defaults) + self.commandConfig = dict(argspecs=command_patterns, defaultArgs=command_defaults) def add_command(self, incomingCommandKls): @@ -191,5 +217,21 @@ def add_command(self, incomingCommandKls): self.command_classes = without_incoming self.setup_from_command_kls_list() + def add_analysis(self, analysis_obj): + self.stats.add_analysis(analysis_obj) + self.summaryDf = df_to_obj(self.stats.presentation_sdf, self.stats.col_order) + #just trigger redisplay + self.update_based_on_df_config(3) + +class Unused(): + def update_based_on_df_config(self, change): + if self.dfConfig['reorderdColumns']: + #ideally this won't require a reserialization. All + #possible col_orders shoudl be serialized once, and the + #frontend should just toggle from them + #self.origDf = df_to_obj(tdf, self.stats.col_order, table_hints=self.stats.table_hints) + self.origDf = df_to_obj(self.typed_df, self.stats.col_order, table_hints=self.stats.table_hints) + else: + self.origDf = df_to_obj(tdf, tdf.columns, table_hints=self.stats.table_hints) diff --git a/buckaroo/cleaning_commands.py b/buckaroo/cleaning_commands.py new file mode 100644 index 00000000..5a1f6853 --- /dev/null +++ b/buckaroo/cleaning_commands.py @@ -0,0 +1,102 @@ +from .lispy import s +from .configure_utils import configure_buckaroo +from .auto_clean import smart_to_int, get_auto_type_operations +import pandas as pd +import numpy as np + +class Command(object): + pass + +class to_bool(Command): + #argument_names = ["df", "col"] + command_default = [s('to_bool'), s('df'), "col"] + command_pattern = [None] + + @staticmethod + def transform(df, col): + ser = df[col] + df[col] = pd.to_numeric(ser, errors='coerce').dropna().astype('boolean').reindex(ser.index) + return df + + @staticmethod + def transform_to_py(df, col): + return " df['%s'] = pd.to_numeric(df['%s'], errors='coerce').dropna().astype('boolean').reindex(df['%s'].index)" % (col, col, col) + +class to_datetime(Command): + #argument_names = ["df", "col"] + command_default = [s('to_datetime'), s('df'), "col"] + command_pattern = [None] + + @staticmethod + def transform(df, col): + ser = df[col] + df[col] = pd.to_datetime(ser, errors='coerce').reindex(ser.index) + return df + + @staticmethod + def transform_to_py(df, col): + return " df['%s'] = pd.to_datetime(df['%s'], errors='coerce').reindex(df['%s'].index)" % (col, col, col) + +class to_int(Command): + #argument_names = ["df", "col"] + command_default = [s('to_int'), s('df'), "col"] + command_pattern = [None] + + @staticmethod + def transform(df, col): + ser = df[col] + try: + df[col] = smart_to_int(ser) + except Exception as e: + #just let pandas figure it out, we recommended the wrong type + df[col] = pd.to_numeric(ser, errors='coerce') + + return df + + @staticmethod + def transform_to_py(df, col): + return " df['%s'] = smart_int(df['%s'])" % (col, col) + +class to_float(Command): + #argument_names = ["df", "col"] + command_default = [s('to_float'), s('df'), "col"] + command_pattern = [None] + + @staticmethod + def transform(df, col): + ser = df[col] + df[col] = pd.to_numeric(ser, errors='coerce').dropna().astype('float').reindex(ser.index) + return df + + @staticmethod + def transform_to_py(df, col): + return " df['%s'] = pd.to_numeric(df['%s'], errors='coerce')" % (col, col) + +class to_string(Command): + #argument_names = ["df", "col"] + command_default = [s('to_string'), s('df'), "col"] + command_pattern = [None] + + @staticmethod + def transform(df, col): + ser = df[col] + df[col] = ser.fillna(value="").astype('string').replace("", None).reindex(ser.index) + return df + + @staticmethod + def transform_to_py(df, col): + return " df['%s'] = df['%s'].fillna(value='').astype('string').replace('', None)" % (col, col) + + + +cleaning_classes = [to_bool, to_datetime, to_int, to_float, to_string,] + +def auto_type_df2(df): + _command_defaults, _command_patterns, transform, buckaroo_to_py_core = configure_buckaroo( + cleaning_classes) + + cleaning_operations = get_auto_type_operations(df) + + full_ops = [{'symbol': 'begin'}] + full_ops.extend(cleaning_operations) + return transform(full_ops, df) diff --git a/buckaroo/configure_utils.py b/buckaroo/configure_utils.py index 219874db..b47f9498 100644 --- a/buckaroo/configure_utils.py +++ b/buckaroo/configure_utils.py @@ -17,7 +17,6 @@ def configure_buckaroo(transforms): def buckaroo_transform(instructions, df): df_copy = df.copy() ret_val = buckaroo_eval(instructions, {'df':df_copy}) - #print(ret_val) return ret_val convert_to_python, __unused = make_interpreter(to_py_lisp_primitives) @@ -28,8 +27,6 @@ def buckaroo_to_py(instructions): #interpreter as 'begin'... that way the exact same instructions #could be sent to either interpreter. For now, this will do individual_instructions = [x for x in map(lambda x:convert_to_python(x, {'df':5}), instructions)] - #print("individual_instructions", individual_instructions) code_block = '\n'.join(individual_instructions) - return "def clean(df):\n" + code_block + "\n return df" return command_defaults, command_patterns, buckaroo_transform, buckaroo_to_py diff --git a/buckaroo/down_sample.py b/buckaroo/down_sample.py index 2ebe5c88..b168f845 100644 --- a/buckaroo/down_sample.py +++ b/buckaroo/down_sample.py @@ -2,6 +2,14 @@ import numpy as np +def get_outlier_idxs(ser): + if not pd.api.types.is_numeric_dtype(ser.dtype): + return [] + outlier_idxs = [] + outlier_idxs.extend(ser.nlargest(5).index) + outlier_idxs.extend(ser.nsmallest(5).index) + return outlier_idxs + def sample(df, sample_size=500, include_outliers=True): if len(df) <= sample_size: diff --git a/buckaroo/lisp_utils.py b/buckaroo/lisp_utils.py new file mode 100644 index 00000000..08a0b658 --- /dev/null +++ b/buckaroo/lisp_utils.py @@ -0,0 +1,46 @@ +""" +It would be awesome to have cleaning and verification commands that add new columns with related names + +The new columns are null accept for errored values. + +Could use this to show original values that were removed/cleaned. Combined with conditional styling in the UI + +sick. And still ahve high perfromance properly typed columns + + + +""" + +def is_symbol(obj): + return isinstance(obj, dict) and "symbol" in obj + +def is_generated_symbol(obj): + return is_symbol(obj) and obj.get("meta", False) + +def split_operations(full_operations): + """ + utitlity to split a combined set of operations with machine generated commands and user entered commands into two lists, machine_generated and user_generated + + machine_generated commands have function calls with the symbol token also having a meta key with a value of {"precleaning":True} + """ + + machine_generated, user_entered = [], [] + for command in full_operations: + assert isinstance(command, list) + sym_atom = command[0] + if is_symbol(sym_atom): + if is_generated_symbol(sym_atom): + machine_generated.append(command) + else: + user_entered.append(command) + continue + raise Exception("Unexpected token %r" % command) + return machine_generated, user_entered + +def lists_match(l1, l2): + #https://note.nkmk.me/en/python-list-compare/#checking-the-exact-match-of-lists + if len(l1) != len(l2): + return False + return all(x == y and type(x) == type(y) for x, y in zip(l1, l2)) + + diff --git a/buckaroo/lispy.py b/buckaroo/lispy.py index 27d58939..b5e740c5 100644 --- a/buckaroo/lispy.py +++ b/buckaroo/lispy.py @@ -186,7 +186,6 @@ def eval(x, env=global_env): eval(exp, env) x = x[-1] else: # (proc exp*) - print("exp", x) exps = [eval(exp, env) for exp in x] proc = exps.pop(0) if isa(proc, Procedure): @@ -197,6 +196,22 @@ def eval(x, env=global_env): return proc(*exps) + def is_unparsed_atom_a_symbol(obj): + if isinstance(obj, dict): + if obj.get('symbol', False): + if len(obj) == 1: + return True + elif len(obj) == 2 and obj.get('meta', False) is not False: + #our symbols can have a meta key too + return True + return False + + def is_unparsed_atom_a_quote(obj): + if isinstance(obj, dict) and len(obj) == 1: + if obj.get('quote', False) is not False: + return True + return False + def list_parse(lst): ret_list = [] if isinstance(lst, list) == False: @@ -207,15 +222,12 @@ def list_parse(lst): while True: if isinstance(x, list): ret_list.append(list_parse(x)) - elif isinstance(x, dict) and len(x) == 1: #hack to make the aprser easier - if x.get('symbol', False): - ret_list.append(Sym(x['symbol'])) - elif x.get('quote', False): - quote_char = x.get('quote') - quote_func = quotes[quote_char] - ret_list.append([quote_func, list_parse(next(lst))]) - else: - ret_list.append(x) + elif is_unparsed_atom_a_symbol(x): + ret_list.append(Sym(x['symbol'])) + elif is_unparsed_atom_a_quote(x): + quote_char = x.get('quote') + quote_func = quotes[quote_char] + ret_list.append([quote_func, list_parse(next(lst))]) elif isinstance(x, dict): print("x was a dict", x) ret_list.append(x) diff --git a/buckaroo/order_columns.py b/buckaroo/order_columns.py index 0b4ac8d6..f5438346 100644 --- a/buckaroo/order_columns.py +++ b/buckaroo/order_columns.py @@ -82,14 +82,6 @@ def reorder_columns(df): col_order = order_columns(tdf_stats, cpd) return df[col_order] -def get_outlier_idxs(ser): - if not pd.api.types.is_numeric_dtype(ser.dtype): - return [] - outlier_idxs = [] - outlier_idxs.extend(ser.nlargest(5).index) - outlier_idxs.extend(ser.nsmallest(5).index) - return outlier_idxs - def add_col_rankings(df, sdf): sdf.loc['one_distinct'] = 0 diff --git a/buckaroo/widget_utils.py b/buckaroo/widget_utils.py index b33493bf..07db0d15 100644 --- a/buckaroo/widget_utils.py +++ b/buckaroo/widget_utils.py @@ -3,7 +3,7 @@ def _display_as_buckaroo(df): from IPython.display import display - return display(BuckarooWidget(df, showCommands=False, showTransformed=False)) + return display(BuckarooWidget(df, showCommands=False)) def enable(): """ diff --git a/docs/source/_static/embed-bundle.js.LICENSE.txt b/docs/source/_static/embed-bundle.js.LICENSE.txt index cfe1226b..6ef8448c 100644 --- a/docs/source/_static/embed-bundle.js.LICENSE.txt +++ b/docs/source/_static/embed-bundle.js.LICENSE.txt @@ -1,3 +1,151 @@ +/*! + Copyright (c) 2016 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames +*/ + +/*! Conditions:: INITIAL */ + +/*! Production:: css_value : ANGLE */ + +/*! Production:: css_value : CHS */ + +/*! Production:: css_value : EMS */ + +/*! Production:: css_value : EXS */ + +/*! Production:: css_value : FREQ */ + +/*! Production:: css_value : LENGTH */ + +/*! Production:: css_value : PERCENTAGE */ + +/*! Production:: css_value : REMS */ + +/*! Production:: css_value : RES */ + +/*! Production:: css_value : SUB css_value */ + +/*! Production:: css_value : TIME */ + +/*! Production:: css_value : VHS */ + +/*! Production:: css_value : VMAXS */ + +/*! Production:: css_value : VMINS */ + +/*! Production:: css_value : VWS */ + +/*! Production:: css_variable : CSS_VAR LPAREN CSS_CPROP COMMA math_expression RPAREN */ + +/*! Production:: css_variable : CSS_VAR LPAREN CSS_CPROP RPAREN */ + +/*! Production:: expression : math_expression EOF */ + +/*! Production:: math_expression : LPAREN math_expression RPAREN */ + +/*! Production:: math_expression : NESTED_CALC LPAREN math_expression RPAREN */ + +/*! Production:: math_expression : SUB PREFIX SUB NESTED_CALC LPAREN math_expression RPAREN */ + +/*! Production:: math_expression : css_value */ + +/*! Production:: math_expression : css_variable */ + +/*! Production:: math_expression : math_expression ADD math_expression */ + +/*! Production:: math_expression : math_expression DIV math_expression */ + +/*! Production:: math_expression : math_expression MUL math_expression */ + +/*! Production:: math_expression : math_expression SUB math_expression */ + +/*! Production:: math_expression : value */ + +/*! Production:: value : NUMBER */ + +/*! Production:: value : SUB NUMBER */ + +/*! Rule:: $ */ + +/*! Rule:: (--[0-9a-z-A-Z-]*) */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)% */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)Hz\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)ch\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)cm\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)deg\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)dpcm\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)dpi\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)dppx\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)em\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)ex\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)grad\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)in\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)kHz\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)mm\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)ms\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)pc\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)pt\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)px\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)rad\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)rem\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)s\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)turn\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)vh\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)vmax\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)vmin\b */ + +/*! Rule:: ([0-9]+(\.[0-9]*)?|\.[0-9]+)vw\b */ + +/*! Rule:: ([a-z]+) */ + +/*! Rule:: (calc) */ + +/*! Rule:: (var) */ + +/*! Rule:: , */ + +/*! Rule:: - */ + +/*! Rule:: \( */ + +/*! Rule:: \) */ + +/*! Rule:: \* */ + +/*! Rule:: \+ */ + +/*! Rule:: \/ */ + +/*! decimal.js-light v2.5.1 https://github.com/MikeMcl/decimal.js-light/LICENCE */ + /** * @ag-grid-community/all-modules - Advanced Data Grid / Data Table supporting Javascript / Typescript / React / Angular / Vue * @version v29.3.5 * @link https://www.ag-grid.com/ @@ -69,3 +217,12 @@ * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. */ + +/** @license React v16.13.1 + * react-is.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ diff --git a/docs/source/articles/histograms.rst b/docs/source/articles/histograms.rst new file mode 100644 index 00000000..4cb9988b --- /dev/null +++ b/docs/source/articles/histograms.rst @@ -0,0 +1,56 @@ +.. _using: + +========== +Histograms +========== + +Buckaroo uses histograms to convey the general shape of a colum in the minimum amount of screen real estate. Like the rest of buckaroo, this is an opionated feature. + + +Simple histograms for numeric columns +===================================== + +Histograms traditionally show the distribution of values in a column allowing different distributions to be identified (normal distribution, bimodal, random, skew). This works well for accurate data without outliers. + +There are a couple types of outliers that mess up normal histogrrams. + +Traditional histograms make no provision for NaNs. There are two ways we could deal with NaN's treating them as another bar, or as an independent bar. We chose a separate bar because NaNs are a property of the entire dataset and the histogram is a function of the relevant values. NaNs are displayed in a different color and pattern. + +Sentinel values. Columns frquently have sentinel values mixed in to convey missing data or some special condition. After dropping NaNs, we then look at the value counts, here is one explanation of a sampling + +imagine a dataset inserts 0 for NaNs, without 0's the range of numbers goes from 55 to 235, 0's account for 10% of observations. 0 is obviosuly a sentinel here and should be plotted as a categorical. If you disagree you can write your own histogram implementation and plug it in with the pluggable analysis framework. + +Extreme values. Buckaroo limits the extents of the set where the histogram is computed so that of the 10 bins, no single bin represents more than 50% of samples, limited to the quantile range from (0.1 to 0.9). The reasoning is that the extreme values represent errant values if they are so far off that they mess up the range of the rest of the histogram. I haven't decided how to convey which quantile range was chosen. + + +Categorical Histograms for everything else +========================================== + +Histograms are generally considered for numeric columns. Most datasets have many categorical or non numeric values, how can we get a quick overview of them? + +Well we already know how to plot NaNs, there are three other sentinel values that matter False, True, and "0". + +Remaining categoricals are plotted as a long tail plot, most frequent on the left with decreasing frequency to the right. The top 7 most frequent values are plotted, with a final bar of "long tail" consisting of the sum of all the remaining values" + + +Objections to this approach +=========================== + +This is not a traditional histogram and should not be read as such. It is the best way to show the most insight about frequency of values in a column that we could come up with. + + +Other research +============== + +https://edwinth.github.io/blog/outlier-bin/ + +https://stackoverflow.com/questions/11882393/matplotlib-disregard-outliers-when-plotting +references + + Boris Iglewicz and David Hoaglin (1993), "Volume 16: How to Detect and + Handle Outliers", The ASQC Basic References in Quality Control: + Statistical Techniques, Edward F. Mykytka, Ph.D., Editor. + + + + diff --git a/examples/App.tsx b/examples/App.tsx index 4c382f9d..fa253994 100644 --- a/examples/App.tsx +++ b/examples/App.tsx @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ -//import 'bootstrap/dist/css/bootstrap.min.css'; +import 'bootstrap/dist/css/bootstrap.min.css'; import React from 'react'; import {HashRouter as Router, Route, Link} from 'react-router-dom'; //import {Button} from 'react-bootstrap'; @@ -16,7 +16,8 @@ const examples = { ColumnsEditorEx: {title: 'ColumnsEditor', file: 'ColumnsEditorEx'}, CommandViewerEx: {title: 'CommandViewer', file: 'CommandViewerEx'}, DFViewerEx: {title: 'DFViewer', file: 'DFViewerEx'}, - StatusBarEx: {title: 'StatusBar', file: 'StatusBarEx'} + StatusBarEx: {title: 'StatusBar', file: 'StatusBarEx'}, + HistogramEx: {title: 'Histogram', file: 'HistogramEx'} }; // The examples use a code-loading technique that I have described in diff --git a/examples/app.css b/examples/app.css index 2802ab69..c464b456 100644 --- a/examples/app.css +++ b/examples/app.css @@ -9,9 +9,10 @@ } .left-menu { - width: 10em; - min-width: 10em; - max-width: 10em; + width: 105; + min-width: 15em; + max-width: 15em; + overflow:hidden } .codeblock { @@ -35,6 +36,9 @@ --color-example: #fff; } +.fluid-container { + min-width:950px +} html,body { @@ -65,3 +69,106 @@ html,body outline:4px solid pink; } +.histogram { + color:black; +} + +.customHeaderRenderer { + height:100%; +} +.customHeaderLabel { + padding:0; +} + + +.histogram-ex { + +} + +.histogram-ex .histogram-wrap {border: 1px solid gray; clear:both} +.histogram-ex span {color:black; width:200px; float:left; text-align:left; } +.histogram-ex .histogram-component {width:105px; float:left} +.patterns { + width: 100vw; + height: 100vw; +} + + +.small-bar { + height:30px; width:10px; +} + +.med-bar { + height:100px; width:100px; +} + +/* +from https://www.magicpattern.design/tools/css-backgrounds +*/ +.pt1 { +background-color: #e5e5f7; +opacity: 0.8; +background-image: repeating-linear-gradient(45deg, #444cf7 25%, transparent 25%, transparent 75%, #444cf7 75%, #444cf7), repeating-linear-gradient(45deg, #444cf7 25%, #e5e5f7 25%, #e5e5f7 75%, #444cf7 75%, #444cf7); +background-position: 0 0, 2px 2px; +background-size: 4px 4px; + +} +.pt4 { + background-color: #e5e5f7; + opacity: 0.8; + background: repeating-linear-gradient( 45deg, #444cf7, #444cf7 3px, #e5e5f7 3px, #e5e5f7 6px ); +} + +.pt5 { +background-color: #e5e5f7; +opacity: 0.8; +background-image: linear-gradient(135deg, #444cf7 25%, transparent 25%), linear-gradient(225deg, #444cf7 25%, transparent 25%), linear-gradient(45deg, #444cf7 25%, transparent 25%), linear-gradient(315deg, #444cf7 25%, #e5e5f7 25%); +background-position: 3px 0, 3px 0, 0 0, 0 0; +background-size: 6px 6px; +background-repeat: repeat; +} + +.pt6 { +background-color: #e5e5f7; +opacity: 0.8; +background-image: radial-gradient(#444cf7 1.2px, #e5e5f7 1.2px); +background-size: 4px 4px; +} + +.pt7 { +background-color: #e5e5f7; +opacity: 0.8; +background-image: radial-gradient( ellipse farthest-corner at 4px 4px , #444cf7, #444cf7 50%, #e5e5f7 50%); +background-size: 4px 4px; + +} +/* +body { + + margin: 0; + min-height: 100vh; +} + +html { + background: #fff; +} + +special values + +true +false +n/a +long tail +completely unique + + +*/ + + +.ag-column-hover: { + overflow: visible +} +.histogram-ex { background:#181d1f} +.histogram-ex span { color : white } + + diff --git a/examples/ex/HistogramEx.tsx b/examples/ex/HistogramEx.tsx new file mode 100644 index 00000000..b65c747a --- /dev/null +++ b/examples/ex/HistogramEx.tsx @@ -0,0 +1,77 @@ +import React from 'react'; +import { HistogramCell} from '../../js/components/CustomHeader'; +import {histograms } from '../../js/components/staticData'; + + + +export default function Simple() { + const { + num_histo, bool_histo, NA_Only, simple_catgeorical, categorical_histo, + categorical_histo_lt, all_unique, unique_na, unique_continuous, + unique_continuous_scaled, unique_continuous_scaled_50, + start_station_categorical} = histograms; + return
+
+ Numeric + +
+
+ Boolean with NA + +
+
+ NA Only + +
+
+ Simple Categorical + +
+ +
+ Categorical unique NA + +
+
+ Categorical_longtail + +
+ +
+ Categorical All unique + +
+ +
+ Categorical Unique with NA + +
+ +
+ Numeric all Unique + +
+
+ start station categorical + +
+ + +
+ Numeric 50% unique + +
+ + + +
+} +/* + + + */ diff --git a/examples/index.tsx b/examples/index.tsx new file mode 100644 index 00000000..221b5b5d --- /dev/null +++ b/examples/index.tsx @@ -0,0 +1,8 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; + +import App from './App'; + +// eslint-disable-next-line no-console +console.debug('React 16/17 mode'); +ReactDOM.render(, document.getElementById('root')); diff --git a/introduction.ipynb b/introduction.ipynb index f7445f3b..f0d1fe22 100644 --- a/introduction.ipynb +++ b/introduction.ipynb @@ -22,25 +22,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1ab9f44b82a345749e10b78c2097dcd4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "BuckarooWidget(commandConfig={'argspecs': {'dropcol': [None], 'to_datetime': [None], 'safeint': [None], 'filln…" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.read_csv('/Users/paddy/code/citibike-play/2014-01 - Citi Bike trip data.csv')\n", "BuckarooWidget(df[:10_000])" @@ -314,10 +298,275 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.98543491, 0.88189975, 0.8519125 , 0.34405232, 0.64923551,\n", + " 0.76498397, 0.08317026, 0.33898759, 0.8959272 , 0.84532194,\n", + " 0.61846565, 0.97065402, 0.33760845, 0.14928914, 0.46931127,\n", + " 0.44402314, 0.3851214 , 0.30878261, 0.15215036, 0.30283917,\n", + " 0.34619956, 0.94479008, 0.52277332, 0.46966805, 0.01990334,\n", + " 0.0090225 , 0.18186927, 0.67424679, 0.38775559, 0.48266829,\n", + " 0.83345845, 0.7639416 , 0.71366249, 0.40440437, 0.0687034 ,\n", + " 0.99350699, 0.02384897, 0.61694475, 0.16986129, 0.76552384,\n", + " 0.49479425, 0.50517121, 0.80248113, 0.38342123, 0.25053957,\n", + " 0.09369322, 0.53027412, 0.80884121, 0.96754405, 0.10643695,\n", + " 0.30732228, 0.09244387, 0.75280274, 0.66100238, 0.21485027,\n", + " 0.74945128, 0.45370822, 0.88729706, 0.18465771, 0.01511092,\n", + " 0.13943961, 0.68186614, 0.68525074, 0.90057088, 0.58703984,\n", + " 0.47070748, 0.85631468, 0.48696279, 0.94382412, 0.91341682,\n", + " 0.85296105, 0.05317179, 0.68182398, 0.13682457, 0.43423833,\n", + " 0.98276478, 0.26740801, 0.73420044, 0.80780702, 0.476105 ,\n", + " 0.587292 , 0.00186236, 0.81152695, 0.42044342, 0.13466556,\n", + " 0.76170374, 0.52000642, 0.93240774, 0.97551015, 0.65619746,\n", + " 0.81113788, 0.90734312, 0.76348578, 0.15419325, 0.56967834,\n", + " 0.89901476, 0.87296504, 0.6050323 , 0.87922915, 0.58874466])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.random.rand(1,100)[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.12371643, 0.27411337, 0.44275076, 0.21505491, 0.57833314])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.random.random_sample(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAySElEQVR4nO3df3RU5Z3H8c8kJBOCTEKgSUgbMK1W+aVYUjD442gJCUitWrZtbNbGlgPVJu3SeFDSFeSHLRJZiqRUym6Fdhdqa7eiRYRMwRqrIUAk5WfRbkFUdpLdxhCBMhmSZ//g5K5jYkjoTDLP5P06Jyfe53nuvd9vZiZ8nJk7cRljjAAAACwS09cFAAAA9BQBBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgnQF9XUC4tLW16eTJkxo8eLBcLldflwMAALrBGKP3339fGRkZion56OdZojbAnDx5UpmZmX1dBgAAuARvv/22PvGJT3zkfNQGmMGDB0u68APweDxhO08gEFBlZaXy8vIUFxcXtvNEEnruHz1L/bNveqbnaGVLz83NzcrMzHT+Hf8oURtg2l828ng8YQ8wiYmJ8ng8EX2HCCV67h89S/2zb3qm52hlW88Xe/sHb+IFAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsM6Avi4AAC7F5fNfCMtx3bFG5ROlsYu2y9/qCumxjz82I6THA/oznoEBAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWKfHAaaqqkq33367MjIy5HK5tHnz5o9ce99998nlcmnVqlVB442NjSosLJTH41FycrJmzZql06dPB63Zv3+/brrpJiUkJCgzM1Pl5eU9LRUAAESpHgeYM2fO6Nprr9WaNWu6XPfss89q165dysjI6DBXWFioQ4cOyev1asuWLaqqqtKcOXOc+ebmZuXl5WnkyJGqra3V448/rkWLFmndunU9LRcAAEShAT3dYfr06Zo+fXqXa9599119+9vf1vbt2zVjxoyguSNHjmjbtm3as2ePsrOzJUkVFRW67bbbtGLFCmVkZGjjxo1qaWnRU089pfj4eI0ZM0Z1dXVauXJlUNABAAD9U48DzMW0tbXpnnvu0bx58zRmzJgO89XV1UpOTnbCiyTl5uYqJiZGNTU1uuuuu1RdXa2bb75Z8fHxzpr8/HwtX75c7733noYMGdLhuH6/X36/39lubm6WJAUCAQUCgVC2GKT92OE8R6Sh5/4jkvt2x5rwHDfGBH0PpUj8OUqRfTuHCz1Hru7WF/IAs3z5cg0YMEDf+c53Op33+XxKTU0NLmLAAKWkpMjn8zlrsrKygtakpaU5c50FmGXLlmnx4sUdxisrK5WYmHhJvfSE1+sN+zkiDT33H5HYd/nE8B5/aXZbyI+5devWkB8zlCLxdg43eo48Z8+e7da6kAaY2tpaPfHEE3r99dflcrlCeeiLKisrU2lpqbPd3NyszMxM5eXlyePxhO28gUBAXq9XU6dOVVxcXNjOE0nouX/0LEV232MXbQ/Lcd0xRkuz27Rgb4z8baH9PXZwUX5IjxcqkXw7hws9R27P7a+gXExIA8wrr7yihoYGjRgxwhlrbW3VAw88oFWrVun48eNKT09XQ0ND0H7nz59XY2Oj0tPTJUnp6emqr68PWtO+3b7mw9xut9xud4fxuLi4Xrmheus8kYSe+49I7NvfGt7/SfK3uUJ+jkj7GX5YJN7O4UbPkae7tYX0c2Duuece7d+/X3V1dc5XRkaG5s2bp+3bL/zfUk5OjpqamlRbW+vst3PnTrW1tWnSpEnOmqqqqqDXwbxer6666qpOXz4CAAD9S4+fgTl9+rT+/Oc/O9vHjh1TXV2dUlJSNGLECA0dOjRofVxcnNLT03XVVVdJkkaNGqVp06Zp9uzZWrt2rQKBgEpKSlRQUOBccv3Vr35Vixcv1qxZs/TQQw/p4MGDeuKJJ/TDH/7w7+kVAABEiR4HmL179+rWW291ttvfd1JUVKQNGzZ06xgbN25USUmJpkyZopiYGM2cOVOrV6925pOSklRZWani4mJNmDBBw4YN08KFC7mEGgAASLqEAHPLLbfImO5fXnj8+PEOYykpKdq0aVOX+11zzTV65ZVXeloeAADoB/hbSAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYp8cBpqqqSrfffrsyMjLkcrm0efNmZy4QCOihhx7SuHHjNGjQIGVkZOhrX/uaTp48GXSMxsZGFRYWyuPxKDk5WbNmzdLp06eD1uzfv1833XSTEhISlJmZqfLy8kvrEAAARJ0eB5gzZ87o2muv1Zo1azrMnT17Vq+//roWLFig119/Xb/5zW909OhRfeELXwhaV1hYqEOHDsnr9WrLli2qqqrSnDlznPnm5mbl5eVp5MiRqq2t1eOPP65FixZp3bp1l9AiAACINgN6usP06dM1ffr0TueSkpLk9XqDxn70ox9p4sSJOnHihEaMGKEjR45o27Zt2rNnj7KzsyVJFRUVuu2227RixQplZGRo48aNamlp0VNPPaX4+HiNGTNGdXV1WrlyZVDQAQAA/VOPA0xPnTp1Si6XS8nJyZKk6upqJScnO+FFknJzcxUTE6Oamhrdddddqq6u1s0336z4+HhnTX5+vpYvX6733ntPQ4YM6XAev98vv9/vbDc3N0u68LJWIBAIU3dyjh3Oc0Qaeu4/Irlvd6wJz3FjTND3UIrEn6MU2bdzuNBz5OpufWENMOfOndNDDz2ku+++Wx6PR5Lk8/mUmpoaXMSAAUpJSZHP53PWZGVlBa1JS0tz5joLMMuWLdPixYs7jFdWVioxMTEk/XTlw8889Qf03H9EYt/lE8N7/KXZbSE/5tatW0N+zFCKxNs53Og58pw9e7Zb68IWYAKBgL785S/LGKMnn3wyXKdxlJWVqbS01Nlubm5WZmam8vLynPAUDoFAQF6vV1OnTlVcXFzYzhNJ6Ll/9CxFdt9jF20Py3HdMUZLs9u0YG+M/G2ukB774KL8kB4vVCL5dg4Xeo7cnttfQbmYsASY9vDy1ltvaefOnUEBIj09XQ0NDUHrz58/r8bGRqWnpztr6uvrg9a0b7ev+TC32y23291hPC4urlduqN46TySh5/4jEvv2t4Y2XHQ4fpsr5OeItJ/hh0Xi7Rxu9Bx5ultbyD8Hpj28vPnmm/rd736noUOHBs3n5OSoqalJtbW1ztjOnTvV1tamSZMmOWuqqqqCXgfzer266qqrOn35CAAA9C89DjCnT59WXV2d6urqJEnHjh1TXV2dTpw4oUAgoH/4h3/Q3r17tXHjRrW2tsrn88nn86mlpUWSNGrUKE2bNk2zZ8/W7t279eqrr6qkpEQFBQXKyMiQJH31q19VfHy8Zs2apUOHDumXv/ylnnjiiaCXiAAAQP/V45eQ9u7dq1tvvdXZbg8VRUVFWrRokZ5//nlJ0vjx44P2e+mll3TLLbdIkjZu3KiSkhJNmTJFMTExmjlzplavXu2sTUpKUmVlpYqLizVhwgQNGzZMCxcu5BJqAAAg6RICzC233CJjPvrywq7m2qWkpGjTpk1drrnmmmv0yiuv9LQ8AADQD/C3kAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwTo//mCMA4NJcPv+Fvi6hU+5Yo/KJ0thF2+VvdQXNHX9sRh9VBXSNZ2AAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHX4JF4AH/kJsV19QisA9CWegQEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwTo8DTFVVlW6//XZlZGTI5XJp8+bNQfPGGC1cuFDDhw/XwIEDlZubqzfffDNoTWNjowoLC+XxeJScnKxZs2bp9OnTQWv279+vm266SQkJCcrMzFR5eXnPuwMAAFGpxwHmzJkzuvbaa7VmzZpO58vLy7V69WqtXbtWNTU1GjRokPLz83Xu3DlnTWFhoQ4dOiSv16stW7aoqqpKc+bMceabm5uVl5enkSNHqra2Vo8//rgWLVqkdevWXUKLAAAg2gzo6Q7Tp0/X9OnTO50zxmjVqlV6+OGHdccdd0iSfv7znystLU2bN29WQUGBjhw5om3btmnPnj3Kzs6WJFVUVOi2227TihUrlJGRoY0bN6qlpUVPPfWU4uPjNWbMGNXV1WnlypVBQQcAAPRPPQ4wXTl27Jh8Pp9yc3OdsaSkJE2aNEnV1dUqKChQdXW1kpOTnfAiSbm5uYqJiVFNTY3uuusuVVdX6+abb1Z8fLyzJj8/X8uXL9d7772nIUOGdDi33++X3+93tpubmyVJgUBAgUAglG0GaT92OM8Raeg5+rhjTefjMSboe39Az8Gi9T4f7Y/pztjSc3frC2mA8fl8kqS0tLSg8bS0NGfO5/MpNTU1uIgBA5SSkhK0Jisrq8Mx2uc6CzDLli3T4sWLO4xXVlYqMTHxEjvqPq/XG/ZzRBp6jh7lE7ueX5rd1juFRBB6vmDr1q19UEnvidbHdFciveezZ892a11IA0xfKisrU2lpqbPd3NyszMxM5eXlyePxhO28gUBAXq9XU6dOVVxcXNjOE0noOfp6Hrtoe6fj7hijpdltWrA3Rv42Vy9X1TfoObjng4vy+6iq8Ir2x3RnbOm5/RWUiwlpgElPT5ck1dfXa/jw4c54fX29xo8f76xpaGgI2u/8+fNqbGx09k9PT1d9fX3Qmvbt9jUf5na75Xa7O4zHxcX1yg3VW+eJJPQcPfytXf9D7W9zXXRNtKHnC6Lx/v5B0fqY7kqk99zd2kL6OTBZWVlKT0/Xjh07nLHm5mbV1NQoJydHkpSTk6OmpibV1tY6a3bu3Km2tjZNmjTJWVNVVRX0OpjX69VVV13V6ctHAACgf+lxgDl9+rTq6upUV1cn6cIbd+vq6nTixAm5XC7NnTtXjz76qJ5//nkdOHBAX/va15SRkaE777xTkjRq1ChNmzZNs2fP1u7du/Xqq6+qpKREBQUFysjIkCR99atfVXx8vGbNmqVDhw7pl7/8pZ544omgl4gAAED/1eOXkPbu3atbb73V2W4PFUVFRdqwYYMefPBBnTlzRnPmzFFTU5NuvPFGbdu2TQkJCc4+GzduVElJiaZMmaKYmBjNnDlTq1evduaTkpJUWVmp4uJiTZgwQcOGDdPChQu5hBoAAEi6hABzyy23yJiPvrzQ5XJpyZIlWrJkyUeuSUlJ0aZNm7o8zzXXXKNXXnmlp+UBAIB+gL+FBAAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1Qh5gWltbtWDBAmVlZWngwIH61Kc+paVLl8oY46wxxmjhwoUaPny4Bg4cqNzcXL355ptBx2lsbFRhYaE8Ho+Sk5M1a9YsnT59OtTlAgAAC4U8wCxfvlxPPvmkfvSjH+nIkSNavny5ysvLVVFR4awpLy/X6tWrtXbtWtXU1GjQoEHKz8/XuXPnnDWFhYU6dOiQvF6vtmzZoqqqKs2ZMyfU5QIAAAsNCPUBX3vtNd1xxx2aMWOGJOnyyy/XL37xC+3evVvShWdfVq1apYcfflh33HGHJOnnP/+50tLStHnzZhUUFOjIkSPatm2b9uzZo+zsbElSRUWFbrvtNq1YsUIZGRmhLhsAAFgk5AFm8uTJWrdund544w19+tOf1h//+Ef94Q9/0MqVKyVJx44dk8/nU25urrNPUlKSJk2apOrqahUUFKi6ulrJyclOeJGk3NxcxcTEqKamRnfddVeH8/r9fvn9fme7ublZkhQIBBQIBELdpqP92OE8R6Sh5+jjjjWdj8eYoO/9AT0Hi9b7fLQ/pjtjS8/drS/kAWb+/Plqbm7W1VdfrdjYWLW2tur73/++CgsLJUk+n0+SlJaWFrRfWlqaM+fz+ZSamhpc6IABSklJcdZ82LJly7R48eIO45WVlUpMTPy7+7oYr9cb9nNEGnqOHuUTu55fmt3WO4VEEHq+YOvWrX1QSe+J1sd0VyK957Nnz3ZrXcgDzK9+9Stt3LhRmzZt0pgxY1RXV6e5c+cqIyNDRUVFoT6do6ysTKWlpc52c3OzMjMzlZeXJ4/HE7bzBgIBeb1eTZ06VXFxcWE7TySh5+jreeyi7Z2Ou2OMlma3acHeGPnbXL1cVd+g5+CeDy7K76OqwivaH9OdsaXn9ldQLibkAWbevHmaP3++CgoKJEnjxo3TW2+9pWXLlqmoqEjp6emSpPr6eg0fPtzZr76+XuPHj5ckpaenq6GhIei458+fV2Njo7P/h7ndbrnd7g7jcXFxvXJD9dZ5Igk9Rw9/a9f/UPvbXBddE23o+YJovL9/ULQ+prsS6T13t7aQX4V09uxZxcQEHzY2NlZtbReemszKylJ6erp27NjhzDc3N6umpkY5OTmSpJycHDU1Nam2ttZZs3PnTrW1tWnSpEmhLhkAAFgm5M/A3H777fr+97+vESNGaMyYMdq3b59Wrlypb3zjG5Ikl8uluXPn6tFHH9WVV16prKwsLViwQBkZGbrzzjslSaNGjdK0adM0e/ZsrV27VoFAQCUlJSooKOAKJAAAEPoAU1FRoQULFuhb3/qWGhoalJGRoW9+85tauHChs+bBBx/UmTNnNGfOHDU1NenGG2/Utm3blJCQ4KzZuHGjSkpKNGXKFMXExGjmzJlavXp1qMsFAAAWCnmAGTx4sFatWqVVq1Z95BqXy6UlS5ZoyZIlH7kmJSVFmzZtCnV5AAAgCvC3kAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYJywB5t1339U//uM/aujQoRo4cKDGjRunvXv3OvPGGC1cuFDDhw/XwIEDlZubqzfffDPoGI2NjSosLJTH41FycrJmzZql06dPh6NcAABgmZAHmPfee0833HCD4uLi9OKLL+rw4cP6l3/5Fw0ZMsRZU15ertWrV2vt2rWqqanRoEGDlJ+fr3PnzjlrCgsLdejQIXm9Xm3ZskVVVVWaM2dOqMsFAAAWGhDqAy5fvlyZmZlav369M5aVleX8tzFGq1at0sMPP6w77rhDkvTzn/9caWlp2rx5swoKCnTkyBFt27ZNe/bsUXZ2tiSpoqJCt912m1asWKGMjIxQlw0AACwS8gDz/PPPKz8/X1/60pf08ssv6+Mf/7i+9a1vafbs2ZKkY8eOyefzKTc319knKSlJkyZNUnV1tQoKClRdXa3k5GQnvEhSbm6uYmJiVFNTo7vuuqvDef1+v/x+v7Pd3NwsSQoEAgoEAqFu09F+7HCeI9LQc/Rxx5rOx2NM0Pf+gJ6DRet9Ptof052xpefu1hfyAPOXv/xFTz75pEpLS/W9731Pe/bs0Xe+8x3Fx8erqKhIPp9PkpSWlha0X1pamjPn8/mUmpoaXOiAAUpJSXHWfNiyZcu0ePHiDuOVlZVKTEwMRWtd8nq9YT9HpKHn6FE+sev5pdltvVNIBKHnC7Zu3doHlfSeaH1MdyXSez579my31oU8wLS1tSk7O1s/+MEPJEnXXXedDh48qLVr16qoqCjUp3OUlZWptLTU2W5ublZmZqby8vLk8XjCdt5AICCv16upU6cqLi4ubOeJJPQcfT2PXbS903F3jNHS7DYt2Bsjf5url6vqG/Qc3PPBRfl9VFV4RftjujO29Nz+CsrFhDzADB8+XKNHjw4aGzVqlP7zP/9TkpSeni5Jqq+v1/Dhw5019fX1Gj9+vLOmoaEh6Bjnz59XY2Ojs/+Hud1uud3uDuNxcXG9ckP11nkiCT1HD39r1/9Q+9tcF10Tbej5gmi8v39QtD6muxLpPXe3tpBfhXTDDTfo6NGjQWNvvPGGRo4cKenCG3rT09O1Y8cOZ765uVk1NTXKycmRJOXk5KipqUm1tbXOmp07d6qtrU2TJk0KdckAAMAyIX8G5rvf/a4mT56sH/zgB/ryl7+s3bt3a926dVq3bp0kyeVyae7cuXr00Ud15ZVXKisrSwsWLFBGRobuvPNOSReesZk2bZpmz56ttWvXKhAIqKSkRAUFBVyBBAAAQh9gPvvZz+rZZ59VWVmZlixZoqysLK1atUqFhYXOmgcffFBnzpzRnDlz1NTUpBtvvFHbtm1TQkKCs2bjxo0qKSnRlClTFBMTo5kzZ2r16tWhLhcAAFgo5AFGkj7/+c/r85///EfOu1wuLVmyREuWLPnINSkpKdq0aVM4ygMAAJbjbyEBAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1hnQ1wUA0eby+S/0dQkAEPUIMACAj2RjID/+2Iy+LgG9gJeQAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDphDzCPPfaYXC6X5s6d64ydO3dOxcXFGjp0qC677DLNnDlT9fX1QfudOHFCM2bMUGJiolJTUzVv3jydP38+3OUCAAALhDXA7NmzRz/5yU90zTXXBI1/97vf1W9/+1s988wzevnll3Xy5El98YtfdOZbW1s1Y8YMtbS06LXXXtPPfvYzbdiwQQsXLgxnuQAAwBJhCzCnT59WYWGh/vVf/1VDhgxxxk+dOqWf/vSnWrlypT73uc9pwoQJWr9+vV577TXt2rVLklRZWanDhw/rP/7jPzR+/HhNnz5dS5cu1Zo1a9TS0hKukgEAgCUGhOvAxcXFmjFjhnJzc/Xoo48647W1tQoEAsrNzXXGrr76ao0YMULV1dW6/vrrVV1drXHjxiktLc1Zk5+fr/vvv1+HDh3Sdddd1+F8fr9ffr/f2W5ubpYkBQIBBQKBcLToHP+D3/sDeu6aO9aEu5xe444xQd/7A3q2X3cep/wei1zdrS8sAebpp5/W66+/rj179nSY8/l8io+PV3JyctB4WlqafD6fs+aD4aV9vn2uM8uWLdPixYs7jFdWVioxMfFS2ugRr9cb9nNEGnruXPnEXiikly3NbuvrEnodPdtr69at3V7L77HIc/bs2W6tC3mAefvtt/VP//RP8nq9SkhICPXhP1JZWZlKS0ud7ebmZmVmZiovL08ejyds5w0EAvJ6vZo6dari4uLCdp5IQs9d9zx20fZeqir83DFGS7PbtGBvjPxtrr4up1fQs/09H1yUf9E1/B6L3J7bX0G5mJAHmNraWjU0NOgzn/mMM9ba2qqqqir96Ec/0vbt29XS0qKmpqagZ2Hq6+uVnp4uSUpPT9fu3buDjtt+lVL7mg9zu91yu90dxuPi4nrlhuqt80QSeu6cv9X+fwA+zN/misq+ukLP9urJ7yV+j0We7tYW8jfxTpkyRQcOHFBdXZ3zlZ2drcLCQue/4+LitGPHDmefo0eP6sSJE8rJyZEk5eTk6MCBA2poaHDWeL1eeTwejR49OtQlAwAAy4T8GZjBgwdr7NixQWODBg3S0KFDnfFZs2aptLRUKSkp8ng8+va3v62cnBxdf/31kqS8vDyNHj1a99xzj8rLy+Xz+fTwww+ruLi402dZAABA/xK2q5C68sMf/lAxMTGaOXOm/H6/8vPz9eMf/9iZj42N1ZYtW3T//fcrJydHgwYNUlFRkZYsWdIX5QIAgAjTKwHm97//fdB2QkKC1qxZozVr1nzkPiNHjuzRO8kBAED/wd9CAgAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGCdkAeYZcuW6bOf/awGDx6s1NRU3XnnnTp69GjQmnPnzqm4uFhDhw7VZZddppkzZ6q+vj5ozYkTJzRjxgwlJiYqNTVV8+bN0/nz50NdLgAAsFDIA8zLL7+s4uJi7dq1S16vV4FAQHl5eTpz5oyz5rvf/a5++9vf6plnntHLL7+skydP6otf/KIz39raqhkzZqilpUWvvfaafvazn2nDhg1auHBhqMsFAAAWGhDqA27bti1oe8OGDUpNTVVtba1uvvlmnTp1Sj/96U+1adMmfe5zn5MkrV+/XqNGjdKuXbt0/fXXq7KyUocPH9bvfvc7paWlafz48Vq6dKkeeughLVq0SPHx8aEuGwAAWCTkAebDTp06JUlKSUmRJNXW1ioQCCg3N9dZc/XVV2vEiBGqrq7W9ddfr+rqao0bN05paWnOmvz8fN1///06dOiQrrvuug7n8fv98vv9znZzc7MkKRAIKBAIhKW39uN/8Ht/QM9dc8eacJfTa9wxJuh7f0DP9uvO45TfY5Gru/WFNcC0tbVp7ty5uuGGGzR27FhJks/nU3x8vJKTk4PWpqWlyefzOWs+GF7a59vnOrNs2TItXry4w3hlZaUSExP/3lYuyuv1hv0ckYaeO1c+sRcK6WVLs9v6uoReR8/22rp1a7fX8nss8pw9e7Zb68IaYIqLi3Xw4EH94Q9/COdpJEllZWUqLS11tpubm5WZmam8vDx5PJ6wnTcQCMjr9Wrq1KmKi4sL23kiCT133fPYRdt7qarwc8cYLc1u04K9MfK3ufq6nF5Bz/b3fHBR/kXX8HsscntufwXlYsIWYEpKSrRlyxZVVVXpE5/4hDOenp6ulpYWNTU1BT0LU19fr/T0dGfN7t27g47XfpVS+5oPc7vdcrvdHcbj4uJ65YbqrfNEEnrunL/V/n8APszf5orKvrpCz/bqye8lfo9Fnu7WFvKrkIwxKikp0bPPPqudO3cqKysraH7ChAmKi4vTjh07nLGjR4/qxIkTysnJkSTl5OTowIEDamhocNZ4vV55PB6NHj061CUDAADLhPwZmOLiYm3atEnPPfecBg8e7LxnJSkpSQMHDlRSUpJmzZql0tJSpaSkyOPx6Nvf/rZycnJ0/fXXS5Ly8vI0evRo3XPPPSovL5fP59PDDz+s4uLiTp9lAQAA/UvIA8yTTz4pSbrllluCxtevX697771XkvTDH/5QMTExmjlzpvx+v/Lz8/XjH//YWRsbG6stW7bo/vvvV05OjgYNGqSioiItWbIk1OUCAAALhTzAGHPxy/ASEhK0Zs0arVmz5iPXjBw5skfvJAcAAP0HfwsJAABYhwADAACsQ4ABAADWCfufEgAAoDddPv+Fi65xxxqVT7zwwZOR8Nk3xx+b0dclWIdnYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1uFzYBDRuvN5Dr0h0j4zAgD6O56BAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgHQIMAACwDgEGAABYhwADAACsQ4ABAADWIcAAAADrEGAAAIB1CDAAAMA6BBgAAGAdAgwAALAOAQYAAFiHAAMAAKxDgAEAANYhwAAAAOsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArDOgrwvoypo1a/T444/L5/Pp2muvVUVFhSZOnNjXZVnp8vkvhOxY7lij8onS2EXb5W91hey4ANBfhfJ39EcJ9e/u44/NCEFVly5in4H55S9/qdLSUj3yyCN6/fXXde211yo/P18NDQ19XRoAAOhjERtgVq5cqdmzZ+vrX/+6Ro8erbVr1yoxMVFPPfVUX5cGAAD6WES+hNTS0qLa2lqVlZU5YzExMcrNzVV1dXWn+/j9fvn9fmf71KlTkqTGxkYFAoGw1RoIBHT27Fn99a9/VVxcXNjO8/cacP5M6I7VZnT2bJsGBGLU2tY/XkLqjz1L/bNveqbnaBXqnv/617+GoKqO3n//fUmSMabrhSYCvfvuu0aSee2114LG582bZyZOnNjpPo888oiRxBdffPHFF198RcHX22+/3WVWiMhnYC5FWVmZSktLne22tjY1NjZq6NChcrnCl66bm5uVmZmpt99+Wx6PJ2zniST03D96lvpn3/RMz9HKlp6NMXr//feVkZHR5bqIDDDDhg1TbGys6uvrg8br6+uVnp7e6T5ut1tutztoLDk5OVwlduDxeCL6DhEO9Nx/9Me+6bl/oOfIlJSUdNE1Efkm3vj4eE2YMEE7duxwxtra2rRjxw7l5OT0YWUAACASROQzMJJUWlqqoqIiZWdna+LEiVq1apXOnDmjr3/9631dGgAA6GMRG2C+8pWv6H/+53+0cOFC+Xw+jR8/Xtu2bVNaWlpflxbE7XbrkUce6fDyVTSj5/6jP/ZNz/0DPdvPZczFrlMCAACILBH5HhgAAICuEGAAAIB1CDAAAMA6BBgAAGAdAswl8Pv9Gj9+vFwul+rq6oLm9u/fr5tuukkJCQnKzMxUeXl5h/2feeYZXX311UpISNC4ceO0devWXqq8577whS9oxIgRSkhI0PDhw3XPPffo5MmTQWuiqefjx49r1qxZysrK0sCBA/WpT31KjzzyiFpaWoLWRVPPkvT9739fkydPVmJi4kd+AOSJEyc0Y8YMJSYmKjU1VfPmzdP58+eD1vz+97/XZz7zGbndbl1xxRXasGFD+IsPoTVr1ujyyy9XQkKCJk2apN27d/d1SZesqqpKt99+uzIyMuRyubR58+ageWOMFi5cqOHDh2vgwIHKzc3Vm2++GbSmsbFRhYWF8ng8Sk5O1qxZs3T69Ole7KJnli1bps9+9rMaPHiwUlNTdeedd+ro0aNBa86dO6fi4mINHTpUl112mWbOnNnhQ1O7c1+PFE8++aSuueYa58PpcnJy9OKLLzrz0dZvkJD88aJ+5jvf+Y6ZPn26kWT27dvnjJ86dcqkpaWZwsJCc/DgQfOLX/zCDBw40PzkJz9x1rz66qsmNjbWlJeXm8OHD5uHH37YxMXFmQMHDvRBJxe3cuVKU11dbY4fP25effVVk5OTY3Jycpz5aOv5xRdfNPfee6/Zvn27+a//+i/z3HPPmdTUVPPAAw84a6KtZ2OMWbhwoVm5cqUpLS01SUlJHebPnz9vxo4da3Jzc82+ffvM1q1bzbBhw0xZWZmz5i9/+YtJTEw0paWl5vDhw6aiosLExsaabdu29WInl+7pp5828fHx5qmnnjKHDh0ys2fPNsnJyaa+vr6vS7skW7duNf/8z/9sfvOb3xhJ5tlnnw2af+yxx0xSUpLZvHmz+eMf/2i+8IUvmKysLPO3v/3NWTNt2jRz7bXXml27dplXXnnFXHHFFebuu+/u5U66Lz8/36xfv94cPHjQ1NXVmdtuu82MGDHCnD592llz3333mczMTLNjxw6zd+9ec/3115vJkyc78925r0eS559/3rzwwgvmjTfeMEePHjXf+973TFxcnDl48KAxJvr6/SACTA9t3brVXH311ebQoUMdAsyPf/xjM2TIEOP3+52xhx56yFx11VXO9pe//GUzY8aMoGNOmjTJfPOb3wx77aHw3HPPGZfLZVpaWowx/aPn8vJyk5WV5WxHc8/r16/vNMBs3brVxMTEGJ/P54w9+eSTxuPxOD+HBx980IwZMyZov6985SsmPz8/rDWHysSJE01xcbGz3draajIyMsyyZcv6sKrQ+HCAaWtrM+np6ebxxx93xpqamozb7Ta/+MUvjDHGHD582Egye/bscda8+OKLxuVymXfffbfXav97NDQ0GEnm5ZdfNsZc6DEuLs4888wzzpojR44YSaa6utoY0737eqQbMmSI+bd/+7eo75eXkHqgvr5es2fP1r//+78rMTGxw3x1dbVuvvlmxcfHO2P5+fk6evSo3nvvPWdNbm5u0H75+fmqrq4Ob/Eh0NjYqI0bN2ry5MmKi4uTFP09S9KpU6eUkpLibPeHnj+surpa48aNC/ogyfz8fDU3N+vQoUPOGlt7bmlpUW1tbVD9MTExys3NtaL+njp27Jh8Pl9Qv0lJSZo0aZLTb3V1tZKTk5Wdne2syc3NVUxMjGpqanq95ktx6tQpSXIev7W1tQoEAkF9X3311RoxYkRQ3xe7r0eq1tZWPf300zpz5oxycnKivl8CTDcZY3TvvffqvvvuC3pAf5DP5+vwScHt2z6fr8s17fOR6KGHHtKgQYM0dOhQnThxQs8995wzF609t/vzn/+siooKffOb33TGor3nzvw9PTc3N+tvf/tb7xR6if73f/9Xra2tUXWbdaW9p6769fl8Sk1NDZofMGCAUlJSrPiZtLW1ae7cubrhhhs0duxYSRd6io+P7/A+rw/3fbH7eqQ5cOCALrvsMrndbt1333169tlnNXr06Kjtt12/DzDz58+Xy+Xq8utPf/qTKioq9P7776usrKyvS/67dbfndvPmzdO+fftUWVmp2NhYfe1rX5Ox7AOce9qzJL377ruaNm2avvSlL2n27Nl9VPmlu5SegWhRXFysgwcP6umnn+7rUsLuqquuUl1dnWpqanT//ferqKhIhw8f7uuywi5i/xZSb3nggQd07733drnmk5/8pHbu3Knq6uoOf0MiOztbhYWF+tnPfqb09PQO7+5u305PT3e+d7amfb43dLfndsOGDdOwYcP06U9/WqNGjVJmZqZ27dqlnJycqO355MmTuvXWWzV58mStW7cuaF209tyV9PT0DlfkdLdnj8ejgQMHdrPqvjFs2DDFxsb2+W3WW9p7qq+v1/Dhw53x+vp6jR8/3lnT0NAQtN/58+fV2NgY8T+TkpISbdmyRVVVVfrEJz7hjKenp6ulpUVNTU1Bz0p88Hbuzn090sTHx+uKK66QJE2YMEF79uzRE088oa985StR2a+jr9+EY4u33nrLHDhwwPnavn27kWR+/etfm7ffftsY8/9v7mx/g6sxxpSVlXV4c+fnP//5oGPn5ORE/Js727311ltGknnppZeMMdHZ8zvvvGOuvPJKU1BQYM6fP99hPhp7bnexN/F+8Iqcn/zkJ8bj8Zhz584ZYy68iXfs2LFB+919991WvYm3pKTE2W5tbTUf//jHo/pNvCtWrHDGTp061embePfu3eus2b59e0S/ibetrc0UFxebjIwM88Ybb3SYb39T669//Wtn7E9/+lOnb2rt6r4e6W699VZTVFQU9f0SYC7RsWPHOlyF1NTUZNLS0sw999xjDh48aJ5++mmTmJjY4fLaAQMGmBUrVpgjR46YRx55JGIvr921a5epqKgw+/btM8ePHzc7duwwkydPNp/61KecO3a09fzOO++YK664wkyZMsW888475r//+7+dr3bR1rMxF4Lpvn37zOLFi81ll11m9u3bZ/bt22fef/99Y8z/X2qZl5dn6urqzLZt28zHPvaxTi+jnjdvnjly5IhZs2aNdZdRu91us2HDBnP48GEzZ84ck5ycHHR1hk3ef/9953aUZFauXGn27dtn3nrrLWPMhcuok5OTzXPPPWf2799v7rjjjk4vo77uuutMTU2N+cMf/mCuvPLKiL6M+v777zdJSUnm97//fdBj9+zZs86a++67z4wYMcLs3LnT7N27t8NHQ3Tnvh5J5s+fb15++WVz7Ngxs3//fjN//nzjcrlMZWWlMSb6+v0gAswl6izAGGPMH//4R3PjjTcat9ttPv7xj5vHHnusw76/+tWvzKc//WkTHx9vxowZY1544YVeqrpn9u/fb2699VaTkpJi3G63ufzyy819991n3nnnnaB10dTz+vXrjaROvz4omno2xpiioqJOe25/ps0YY44fP26mT59uBg4caIYNG2YeeOABEwgEgo7z0ksvmfHjx5v4+HjzyU9+0qxfv753G/k7VVRUmBEjRpj4+HgzceJEs2vXrr4u6ZK99NJLnd6mRUVFxpgLz1YsWLDApKWlGbfbbaZMmWKOHj0adIy//vWv5u677zaXXXaZ8Xg85utf/7oTaiPRRz12P3g//Nvf/ma+9a1vmSFDhpjExERz1113Bf0PijHdu69Him984xtm5MiRJj4+3nzsYx8zU6ZMccKLMdHX7we5jLHs3ZgAAKDf6/dXIQEAAPsQYAAAgHUIMAAAwDoEGAAAYB0CDAAAsA4BBgAAWIcAAwAArEOAAQAA1iHAAAAA6xBgAACAdQgwAADAOgQYAABgnf8D9iExpIWQKPsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "arr = np.random.standard_normal(5000) * 100\n", + "arr = arr.astype(int)\n", + "ser = pd.Series(arr)\n", + "ser.hist()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([ 3, 17, 124, 501, 1079, 1483, 1123, 497, 156, 17])" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "populations, endpoints = np.histogram(arr, 10)\n", + "populations" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([-406. , -332.3, -258.6, -184.9, -111.2, -37.5, 36.2, 109.9,\n", + " 183.6, 257.3, 331. ])" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "endpoints" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['-406 -332',\n", + " '-332 -258',\n", + " '-258 -184',\n", + " '-184 -111',\n", + " '-111 -37',\n", + " '-37 36',\n", + " '36 109',\n", + " '109 183',\n", + " '183 257',\n", + " '257 331']" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def histogram_labels(endpoints):\n", + " left = endpoints[0]\n", + " labels = []\n", + " for edge in endpoints[1:]:\n", + " labels.append(\"%d %d\" % (left, edge))\n", + " left = edge\n", + " return labels\n", + "histogram_labels(endpoints)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.0006, 0.0034, 0.0248, 0.1002, 0.2158, 0.2966, 0.2246, 0.0994,\n", + " 0.0312, 0.0034])" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "normalized_pop = populations / populations.sum()\n", + "normalized_pop" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'-406 -332': 0.0006,\n", + " '-332 -258': 0.0034,\n", + " '-258 -184': 0.0248,\n", + " '-184 -111': 0.1002,\n", + " '-111 -37': 0.2158,\n", + " '-37 36': 0.2966,\n", + " '36 109': 0.2246,\n", + " '109 183': 0.0994,\n", + " '183 257': 0.0312,\n", + " '257 331': 0.0034}" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dict(zip( histogram_labels(endpoints), normalized_pop))" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': '-406 -332', 'population': 0.0006},\n", + " {'name': '-332 -258', 'population': 0.0034},\n", + " {'name': '-258 -184', 'population': 0.0248},\n", + " {'name': '-184 -111', 'population': 0.1002},\n", + " {'name': '-111 -37', 'population': 0.2158},\n", + " {'name': '-37 36', 'population': 0.2966},\n", + " {'name': '36 109', 'population': 0.2246},\n", + " {'name': '109 183', 'population': 0.0994},\n", + " {'name': '183 257', 'population': 0.0312},\n", + " {'name': '257 331', 'population': 0.0034}]" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def histogram_formatted_dict(arr):\n", + " populations, endpoints = np.histogram(arr, 10)\n", + " labels = histogram_labels(endpoints)\n", + " normalized_pop = populations / populations.sum()\n", + " ret_histo = []\n", + " for label, pop in zip(labels, normalized_pop):\n", + " ret_histo.append({'name': label, 'population':pop})\n", + " return ret_histo\n", + "histogram_formatted_dict(arr)" + ] }, { "cell_type": "code", diff --git a/js/components/BaseHeader.tsx b/js/components/BaseHeader.tsx new file mode 100644 index 00000000..ece765f9 --- /dev/null +++ b/js/components/BaseHeader.tsx @@ -0,0 +1,465 @@ +/* +import { Autowired } from "../../../context/context"; +import { Column } from "../../../entities/column"; +import { IComponent } from "../../../interfaces/iComponent"; +import { IMenuFactory } from "../../../interfaces/iMenuFactory"; +import { AgGridCommon } from "../../../interfaces/iCommon"; +import { SortController } from "../../../sortController"; +import { firstExistingValue } from "../../../utils/array"; +import { isIOSUserAgent } from "../../../utils/browser"; +import { removeFromParent, setDisplayed } from "../../../utils/dom"; +import { exists } from "../../../utils/generic"; +import { createIconNoSpan } from "../../../utils/icon"; +import { escapeString } from "../../../utils/string"; +import { Component } from "../../../widgets/component"; +import { RefSelector } from "../../../widgets/componentAnnotations"; +import { LongTapEvent, TapEvent, TouchListener } from "../../../widgets/touchListener"; +import { SortIndicatorComp } from "./sortIndicatorComp"; +import { ColumnModel } from "../../../columns/columnModel"; +import { Events } from "../../../eventKeys"; +import { SortDirection } from "../../../entities/colDef"; + + +import { firstExistingValue } from "../../../utils/array"; +*/ + +import { + Autowired, + Column, + IComponent, + IMenuFactory, + SortController, + _, // contains all of the utils stuff + Component, + RefSelector, + LongTapEvent, + TapEvent, + TouchListener, + SortIndicatorComp, + ColumnModel, + Events, + SortDirection, + GridApi, + ColumnApi, +} from 'ag-grid-community'; + +const escapeString = _.escapeString; +const removeFromParent = _.removeFromParent; +const setDisplayed = _.setDisplayed; +const firstExistingValue = _.firstExistingValue; +const isIOSUserAgent = _.isIOSUserAgent; + +const exists = _.exists; +const createIconNoSpan = _.createIconNoSpan; + +export interface AgGridCommon { + /** The grid api. */ + api: GridApi; + /** The column api. */ + columnApi: ColumnApi; + /** Application context as set on `gridOptions.context`. */ + context: TContext; +} + +export interface IHeaderParams + extends AgGridCommon { + /** The column the header is for. */ + column: Column; + /** + * The name to display for the column. + * If the column is using a headerValueGetter, the displayName will take this into account. + */ + displayName: string; + /** + * Whether sorting is enabled for the column. + * Only put sort logic into your header if this is true. + */ + enableSorting: boolean | undefined; + /** + * Whether menu is enabled for the column. + * Only display a menu button in your header if this is true. + */ + enableMenu: boolean; + /** + * Callback to request the grid to show the column menu. + * Pass in the html element of the column menu to have the + * grid position the menu over the button. + */ + showColumnMenu: (source: HTMLElement) => void; + /** + * Callback to progress the sort for this column. + * The grid will decide the next sort direction eg ascending, descending or 'no sort'. + * Pass `multiSort=true` if you want to do a multi sort (eg user has Shift held down when they click). + */ + progressSort: (multiSort?: boolean) => void; + /** + * Callback to set the sort for this column. + * Pass the sort direction to use ignoring the current sort eg one of 'asc', 'desc' or null (for no sort). + * Pass `multiSort=true` if you want to do a multi sort (eg user has Shift held down when they click) + */ + setSort: (sort: SortDirection, multiSort?: boolean) => void; + + /** Custom header template if provided to `headerComponentParams`, otherwise will be `undefined`. See [Header Templates](https://ag-grid.com/javascript-data-grid/column-headers/#header-templates) */ + template?: string; + /** + * The header the grid provides. + * The custom header component is a child of the grid provided header. + * The grid's header component is what contains the grid managed functionality such as resizing, keyboard navigation etc. + * This is provided should you want to make changes to this cell, + * eg add ARIA tags, or add keyboard event listener (as focus goes here when navigating to the header). + */ + eGridHeader: HTMLElement; +} + +export interface IHeader { + /** Get the header to refresh. Gets called whenever Column Defs are updated. */ + refresh(params: IHeaderParams): boolean; +} + +export interface IHeaderComp extends IHeader, IComponent {} + +export class HeaderComp extends Component implements IHeaderComp { + private static TEMPLATE /* html */ = ``; + + @Autowired('sortController') private sortController: SortController; + @Autowired('menuFactory') private menuFactory: IMenuFactory; + @Autowired('columnModel') private readonly columnModel: ColumnModel; + + @RefSelector('eFilter') private eFilter: HTMLElement; + @RefSelector('eSortIndicator') private eSortIndicator: SortIndicatorComp; + @RefSelector('eMenu') private eMenu: HTMLElement; + @RefSelector('eLabel') private eLabel: HTMLElement; + @RefSelector('eText') private eText: HTMLElement; + + /** + * Selectors for custom headers templates + */ + @RefSelector('eSortOrder') private eSortOrder: HTMLElement; + @RefSelector('eSortAsc') private eSortAsc: HTMLElement; + @RefSelector('eSortDesc') private eSortDesc: HTMLElement; + @RefSelector('eSortMixed') private eSortMixed: HTMLElement; + @RefSelector('eSortNone') private eSortNone: HTMLElement; + + private params: IHeaderParams; + + private lastMovingChanged = 0; + + private currentDisplayName: string; + private currentTemplate: string | null | undefined; + private currentShowMenu: boolean; + private currentSort: boolean | undefined; + + // this is a user component, and IComponent has "public destroy()" as part of the interface. + // so we need to override destroy() just to make the method public. + public destroy(): void { + super.destroy(); + } + + public refresh(params: IHeaderParams): boolean { + this.params = params; + + // if template changed, then recreate the whole comp, the code required to manage + // a changing template is to difficult for what it's worth. + if (this.workOutTemplate() != this.currentTemplate) { + return false; + } + if (this.workOutShowMenu() != this.currentShowMenu) { + return false; + } + if (this.workOutSort() != this.currentSort) { + return false; + } + + this.setDisplayName(params); + + return true; + } + + private workOutTemplate(): string | null | undefined { + let template: string | null | undefined = firstExistingValue( + this.params.template, + HeaderComp.TEMPLATE + ); + + // take account of any newlines & whitespace before/after the actual template + template = template && template.trim ? template.trim() : template; + return template; + } + + public init(params: IHeaderParams): void { + this.params = params; + + this.currentTemplate = this.workOutTemplate(); + this.setTemplate(this.currentTemplate); + this.setupTap(); + this.setupIcons(params.column); + this.setMenu(); + this.setupSort(); + this.setupFilterIcon(); + this.setDisplayName(params); + } + + private setDisplayName(params: IHeaderParams): void { + if (this.currentDisplayName != params.displayName) { + this.currentDisplayName = params.displayName; + const displayNameSanitised = escapeString(this.currentDisplayName); + if (this.eText) { + this.eText.innerHTML = displayNameSanitised!; + } + } + } + + private setupIcons(column: Column): void { + this.addInIcon('menu', this.eMenu, column); + this.addInIcon('filter', this.eFilter, column); + } + + private addInIcon( + iconName: string, + eParent: HTMLElement, + column: Column + ): void { + if (eParent == null) { + return; + } + + const eIcon = createIconNoSpan(iconName, this.gridOptionsService, column); + if (eIcon) { + eParent.appendChild(eIcon); + } + } + + private setupTap(): void { + const { gridOptionsService } = this; + + if (gridOptionsService.is('suppressTouch')) { + return; + } + + const touchListener = new TouchListener(this.getGui(), true); + const suppressMenuHide = gridOptionsService.is('suppressMenuHide'); + const tapMenuButton = suppressMenuHide && exists(this.eMenu); + const menuTouchListener = tapMenuButton + ? new TouchListener(this.eMenu, true) + : touchListener; + + if (this.params.enableMenu) { + const eventType = tapMenuButton ? 'EVENT_TAP' : 'EVENT_LONG_TAP'; + const showMenuFn = (event: TapEvent | LongTapEvent) => { + gridOptionsService.api.showColumnMenuAfterMouseClick( + this.params.column, + event.touchStart + ); + }; + this.addManagedListener( + menuTouchListener, + TouchListener[eventType], + showMenuFn + ); + } + + if (this.params.enableSorting) { + const tapListener = (event: TapEvent) => { + const target = event.touchStart.target as HTMLElement; + // When suppressMenuHide is true, a tap on the menu icon will bubble up + // to the header container, in that case we should not sort + if (suppressMenuHide && this.eMenu.contains(target)) { + return; + } + + this.sortController.progressSort( + this.params.column, + false, + 'uiColumnSorted' + ); + }; + + this.addManagedListener( + touchListener, + TouchListener.EVENT_TAP, + tapListener + ); + } + + // if tapMenuButton is true `touchListener` and `menuTouchListener` are different + // so we need to make sure to destroy both listeners here + this.addDestroyFunc(() => touchListener.destroy()); + + if (tapMenuButton) { + this.addDestroyFunc(() => menuTouchListener.destroy()); + } + } + + private workOutShowMenu(): boolean { + // we don't show the menu if on an iPad/iPhone, as the user cannot have a pointer device/ + // However if suppressMenuHide is set to true the menu will be displayed alwasys, so it's ok + // to show it on iPad in this case (as hover isn't needed). If suppressMenuHide + // is false (default) user will need to use longpress to display the menu. + const menuHides = !this.gridOptionsService.is('suppressMenuHide'); + + const onIpadAndMenuHides = isIOSUserAgent() && menuHides; + const showMenu = this.params.enableMenu && !onIpadAndMenuHides; + + return showMenu; + } + + private setMenu(): void { + // if no menu provided in template, do nothing + if (!this.eMenu) { + return; + } + + this.currentShowMenu = this.workOutShowMenu(); + if (!this.currentShowMenu) { + removeFromParent(this.eMenu); + return; + } + + const suppressMenuHide = this.gridOptionsService.is('suppressMenuHide'); + this.addManagedListener(this.eMenu, 'click', () => + this.showMenu(this.eMenu) + ); + this.eMenu.classList.toggle('ag-header-menu-always-show', suppressMenuHide); + } + + public showMenu(eventSource?: HTMLElement) { + if (!eventSource) { + eventSource = this.eMenu; + } + + this.menuFactory.showMenuAfterButtonClick( + this.params.column, + eventSource, + 'columnMenu' + ); + } + + private workOutSort(): boolean | undefined { + return this.params.enableSorting; + } + + public setupSort(): void { + this.currentSort = this.params.enableSorting; + + // eSortIndicator will not be present when customers provided custom header + // templates, in that case, we need to look for provided sort elements and + // manually create eSortIndicator. + if (!this.eSortIndicator) { + this.eSortIndicator = this.context.createBean( + new SortIndicatorComp(true) + ); + this.eSortIndicator.attachCustomElements( + this.eSortOrder, + this.eSortAsc, + this.eSortDesc, + this.eSortMixed, + this.eSortNone + ); + } + this.eSortIndicator.setupSort(this.params.column); + + // we set up the indicator prior to the check for whether this column is sortable, as it allows the indicator to + // set up the multi sort indicator which can appear irrelevant of whether this column can itself be sorted. + // this can occur in the case of a non-sortable group display column. + if (!this.currentSort) { + return; + } + + const sortUsingCtrl = + this.gridOptionsService.get('multiSortKey') === 'ctrl'; + + // keep track of last time the moving changed flag was set + this.addManagedListener( + this.params.column, + Column.EVENT_MOVING_CHANGED, + () => { + this.lastMovingChanged = new Date().getTime(); + } + ); + + // add the event on the header, so when clicked, we do sorting + if (this.eLabel) { + this.addManagedListener(this.eLabel, 'click', (event: MouseEvent) => { + // sometimes when moving a column via dragging, this was also firing a clicked event. + // here is issue raised by user: https://ag-grid.zendesk.com/agent/tickets/1076 + // this check stops sort if a) column is moving or b) column moved less than 200ms ago (so caters for race condition) + const moving = this.params.column.isMoving(); + const nowTime = new Date().getTime(); + // typically there is <2ms if moving flag was set recently, as it would be done in same VM turn + const movedRecently = nowTime - this.lastMovingChanged < 50; + const columnMoving = moving || movedRecently; + + if (!columnMoving) { + const multiSort = sortUsingCtrl + ? event.ctrlKey || event.metaKey + : event.shiftKey; + this.params.progressSort(multiSort); + } + }); + } + + const onSortingChanged = () => { + this.addOrRemoveCssClass( + 'ag-header-cell-sorted-asc', + this.params.column.isSortAscending() + ); + this.addOrRemoveCssClass( + 'ag-header-cell-sorted-desc', + this.params.column.isSortDescending() + ); + this.addOrRemoveCssClass( + 'ag-header-cell-sorted-none', + this.params.column.isSortNone() + ); + + if (this.params.column.getColDef().showRowGroup) { + const sourceColumns = this.columnModel.getSourceColumnsForGroupColumn( + this.params.column + ); + // this == is intentional, as it allows null and undefined to match, which are both unsorted states + const sortDirectionsMatch = sourceColumns?.every( + (sourceCol) => this.params.column.getSort() == sourceCol.getSort() + ); + const isMultiSorting = !sortDirectionsMatch; + + this.addOrRemoveCssClass('ag-header-cell-sorted-mixed', isMultiSorting); + } + }; + this.addManagedListener( + this.eventService, + Events.EVENT_SORT_CHANGED, + onSortingChanged + ); + this.addManagedListener( + this.eventService, + Events.EVENT_COLUMN_ROW_GROUP_CHANGED, + onSortingChanged + ); + } + + private setupFilterIcon(): void { + if (!this.eFilter) { + return; + } + + this.addManagedListener( + this.params.column, + Column.EVENT_FILTER_CHANGED, + this.onFilterChanged.bind(this) + ); + this.onFilterChanged(); + } + + private onFilterChanged(): void { + const filterPresent = this.params.column.isFilterActive(); + setDisplayed(this.eFilter, filterPresent, { skipAriaHidden: true }); + } +} diff --git a/js/components/ColumnsEditor.tsx b/js/components/ColumnsEditor.tsx index 0b41d678..461dc22c 100644 --- a/js/components/ColumnsEditor.tsx +++ b/js/components/ColumnsEditor.tsx @@ -5,15 +5,14 @@ import { Operation } from './OperationUtils'; import { CommandConfigT } from './CommandUtils'; //import {bakedCommandConfig} from './bakedOperationDefaults'; import { DependentTabs, OperationResult } from './DependentTabs'; -import { tableDf, bakedCommandConfig } from './staticData' +import { tableDf, bakedCommandConfig } from './staticData'; export type OperationSetter = (ops: Operation[]) => void; export interface WidgetConfig { - showCommands:boolean; -// showTransformed:boolean; + showCommands: boolean; + // showTransformed:boolean; } - export function ColumnsEditor({ df, activeColumn, @@ -21,7 +20,7 @@ export function ColumnsEditor({ setOperations, operationResult, commandConfig, - widgetConfig + widgetConfig, }: { df: DFWhole; activeColumn: string; @@ -29,26 +28,29 @@ export function ColumnsEditor({ setOperations: OperationSetter; operationResult: OperationResult; commandConfig: CommandConfigT; - widgetConfig:WidgetConfig + widgetConfig: WidgetConfig; }) { const allColumns = df.schema.fields.map((field) => field.name); //console.log('Columns Editor, commandConfig', commandConfig); return (
- {(widgetConfig.showCommands) ? ( -
- - -
- ) : } + {widgetConfig.showCommands ? ( +
+ + +
+ ) : ( + + )}
); } @@ -69,8 +71,8 @@ export function ColumnsEditorEx() { commandConfig={bakedCommandConfig} operations={operations} setOperations={setOperations} - operationResult={baseOperationResults} - widgetConfig={{showCommands:true}} + operationResult={baseOperationResults} + widgetConfig={{ showCommands: true }} /> ); } diff --git a/js/components/CustomHeader.tsx b/js/components/CustomHeader.tsx new file mode 100644 index 00000000..9fd20d03 --- /dev/null +++ b/js/components/CustomHeader.tsx @@ -0,0 +1,257 @@ +import _ from 'lodash'; +import React from 'react'; +import { createPortal } from 'react-dom'; +import { IHeaderParams } from './BaseHeader'; + +import { + BarChart, + Bar, + //Tooltip, + //Legend, + //Cell, XAxis, YAxis, CartesianGrid, , ResponsiveContainer, +} from 'recharts'; +import { Tooltip } from './RechartTooltip'; +import { isNumOrStr, ValueType } from './RechartExtra'; +//import { ICellRendererParams } from 'ag-grid-community'; + +export interface ICustomHeaderParams extends IHeaderParams { + menuIcon: string; + histogram?: number[]; +} + +function defaultFormatter(value: TValue) { + return _.isArray(value) && isNumOrStr(value[0]) && isNumOrStr(value[1]) + ? (value.join(' ~ ') as TValue) + : value; +} + +export const bakedData = [ + { + name: 'Page A', + population: 4000, + }, + { + name: 'Page B', + population: 3000, + }, + { + name: 'Page C', + population: 2000, + }, + { + name: 'Page D', + population: 2780, + }, + { + name: 'Page E', + population: 1890, + }, +]; + +export const makeData = (histogram: number[]) => { + const accum = []; + for (let i = 0; i < histogram.length; i++) { + accum.push({ + name: `${i + 1}/${histogram.length}`, + population: histogram[i], + }); + } + //console.log('accum', accum) + return accum; +}; + +const formatter = (value: any, name: any, props: any) => { + if (props.payload.name === 'longtail') { + return [value, name]; + } + else { + return [value, props.payload.name]; + } +}; + + +export function FloatingTooltip({ items, x, y }: any) { + const offset = 30; + const renderedItems = items.map((name: [string, number], value: number | string) => { + + const [realName, realValue] = name; + const formattedVal = realValue == 0 ? "<1" : realValue; + return ( + +
{realName}
+
{formattedVal}%
+
+ ); + }); + return createPortal( +
+
{renderedItems}
+
, + document.body + ); +} + +export const ToolTipAdapter = (args: any) => { + const { active, formatter, payload, label } = args; + console.log('label', label); + //console.log("args", args, formatter); + //console.log("coordinate, box", args.coordinate, args.box); + //console.log("payload", payload) + //const anchor = useRef(null); + + if (active && payload && payload.length) { + const renderContent2 = () => { + //const items = (itemSorter ? _.sortBy(payload, itemSorter) : payload).map((entry, i) => { + const items = payload.map((entry: any, i: number) => { + if (entry.type === 'none') { + return null; + } + + const finalFormatter = entry.formatter || formatter || defaultFormatter; + const { value, name } = entry; + let finalValue: React.ReactNode = value; + let finalName: React.ReactNode = name; + if (finalFormatter && finalValue != null && finalName != null) { + const formatted = finalFormatter(value, name, entry, i, payload); + if (Array.isArray(formatted)) { + [finalValue, finalName] = formatted; + } else { + finalValue = formatted; + } + } + + return [finalName, finalValue]; + }); + return items; + }; + /* +

{`${label} : ${payload[0].value}`}

+

{label}

+

Anything you want can be displayed here.

+ + + */ + return ( +
+ +
+ ); + } + + return null; +}; + +//export const HistogramCell = ({histogram}: {histogram:any}) => { +export const HistogramCell = (props: any) => { + if (props == undefined || props.value == undefined) { + return ; + } + const histogram = props.value.histogram; + return ( +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } + offset={20} + allowEscapeViewBox={{ x: true }} + /> + +
+ ); +}; diff --git a/js/components/DCFCell.tsx b/js/components/DCFCell.tsx index 3767ee15..b12d3d40 100644 --- a/js/components/DCFCell.tsx +++ b/js/components/DCFCell.tsx @@ -13,7 +13,6 @@ export type CommandConfigSetterT = ( setter: Dispatch> ) => void; - /* Widget DCFCell is meant to be used with callback functions and passed values, not explicit http calls */ @@ -25,7 +24,7 @@ export function WidgetDCFCell({ commandConfig, dfConfig, on_dfConfig, - summaryDf + summaryDf, }: { origDf: DFWhole; operations: Operation[]; @@ -34,11 +33,11 @@ export function WidgetDCFCell({ commandConfig: CommandConfigT; dfConfig: DfConfig; on_dfConfig: unknown; - summaryDf: DFWhole + summaryDf: DFWhole; }) { const [activeCol, setActiveCol] = useState('stoptime'); - const widgetConfig: WidgetConfig = {showCommands:dfConfig.showCommands} - const localDfConfig = {...dfConfig, 'rowsShown': origDf.data.length || 0} + const widgetConfig: WidgetConfig = { showCommands: dfConfig.showCommands }; + const localDfConfig = { ...dfConfig, rowsShown: origDf.data.length || 0 }; return (
- {(widgetConfig.showCommands) ? ( - ) : } + {widgetConfig.showCommands ? ( + + ) : ( + + )} ); } @@ -90,7 +92,7 @@ export function WidgetDCFCellExample() { commandConfig={bakedCommandConfig} dfConfig={sampleConfig} on_dfConfig={setConfig} - summaryDf={tableDf} + summaryDf={tableDf} /> ); } diff --git a/js/components/DFViewer.tsx b/js/components/DFViewer.tsx index 7d5a585f..30a756d0 100644 --- a/js/components/DFViewer.tsx +++ b/js/components/DFViewer.tsx @@ -1,14 +1,16 @@ import React, { useRef, CSSProperties, + // useMemo, } from 'react'; import _ from 'lodash'; import { DFWhole, EmptyDf } from './staticData'; + import { updateAtMatch, dfToAgrid } from './gridUtils'; import { AgGridReact } from 'ag-grid-react'; // the AG Grid React Component -import { - GridOptions, -} from 'ag-grid-community'; +import { GridOptions } from 'ag-grid-community'; + +import { HistogramCell } from './CustomHeader'; export type setColumFunc = (newCol: string) => void; @@ -44,7 +46,21 @@ export function DFViewer( rowSelection: 'single', onRowClicked: (event) => console.log('A row was clicked'), defaultColDef: { - sortable:true, type: 'rightAligned' + sortable: true, + type: 'rightAligned', + cellRendererSelector: (params) => { + if (params.node.rowPinned) { + return { + component: HistogramCell, + params: { + style: { fontStyle: 'italic' }, + }, + }; + } else { + // rows that are not pinned don't use any cell renderer + return undefined; + } + }, }, onCellClicked: (event) => { @@ -132,6 +148,19 @@ export function DFViewer( }; makeCondtionalAutosize(50, 350); + //const histograms = _.fromPairs( _.map({'a':10, 'b':20}, function(val,key) {return [y, x] })) + // const histograms = _.fromPairs( _.map(df.table_hints, + // function(val,key) { + // return [key, val.histogram || []] })) + const pinnedTopRowData = [df.table_hints]; + //const pinnedTopRowData = [histograms]; + // const pinnedTopRowData = [{'index': 'foo', + // 'end station name': 'bar', + // 'tripduration': 471, + // 'start station name': 'Catherine St & Monroe St', + // 'floatCol': '1.111', + // }]; + return (
diff --git a/js/components/OperationDetail.tsx b/js/components/OperationDetail.tsx index 8fb5de78..ee0e5cf9 100644 --- a/js/components/OperationDetail.tsx +++ b/js/components/OperationDetail.tsx @@ -9,7 +9,6 @@ import { ActualArg, CommandArgSpec } from './CommandUtils'; import { objWithoutNull, replaceAtIdx, replaceAtKey } from './utils'; import React from 'react'; - export const OperationDetail = ({ command, setCommand, @@ -71,7 +70,7 @@ export const ArgGetters = ({ const makeArgGetter = (pattern: ActualArg) => { const idx = pattern[0]; const val = command[idx] as SettableArg; - const valSetter = (newVal:unknown) => { + const valSetter = (newVal: unknown) => { const newCommand = replaceAtIdx(command, idx, newVal); //console.log('newCommand', newCommand); setCommand(newCommand as Operation); @@ -108,8 +107,9 @@ const ArgGetter = ({ }) => { const [_argPos, label, argType, lastArg] = argProps; - const defaultShim = (event: { target: { value: SettableArg; }; }) => setter(event.target.value); - if (argType === 'enum' && _.isArray(lastArg)) { + const defaultShim = (event: { target: { value: SettableArg } }) => + setter(event.target.value); + if (argType === 'enum' && _.isArray(lastArg)) { return (
@@ -124,7 +124,8 @@ const ArgGetter = ({ ); } else if (argType === 'type') { if (lastArg === 'integer') { - const valSetterShim = (event: { target: { value: string; }; }) => setter(parseInt(event.target.value)); + const valSetterShim = (event: { target: { value: string } }) => + setter(parseInt(event.target.value)); return (
@@ -146,7 +147,7 @@ const ArgGetter = ({ } } else if (argType === 'colEnum') { const widgetRow = columns.map((colName: string) => { - const colSetter = (event: { target: { value: any; }; }) => { + const colSetter = (event: { target: { value: any } }) => { const newColVal = event.target.value; if (_.isString(newColVal)) { const updatedColDict = replaceAtKey( @@ -158,9 +159,8 @@ const ArgGetter = ({ } }; const colVal = _.get(val, colName, 'null'); - if(!_.isArray(lastArg)) { - - return

arg error

+ if (!_.isArray(lastArg)) { + return

arg error

; } return ( diff --git a/js/components/OperationUtils.ts b/js/components/OperationUtils.ts index 239c2545..4a2969f1 100644 --- a/js/components/OperationUtils.ts +++ b/js/components/OperationUtils.ts @@ -5,7 +5,6 @@ export const sym = (symbolName: string) => { return { symbol: symbolName }; }; - export type Atom = number | string | SymbolT | ColEnumArgs; export type SettableArg = number | string | ColEnumArgs; diff --git a/js/components/Operations.tsx b/js/components/Operations.tsx index 58c5ce24..7be4e3a0 100644 --- a/js/components/Operations.tsx +++ b/js/components/Operations.tsx @@ -1,7 +1,4 @@ -import React, { - useState, - useEffect, -} from 'react'; +import React, { useState, useEffect } from 'react'; import _ from 'lodash'; import { Operation, @@ -82,7 +79,15 @@ export const OperationsList = ({ ); }; -export const OperationAdder = ({ column, addOperationCb, defaultArgs }:{ column:string, addOperationCb:any, defaultArgs:any } ): JSX.Element => { +export const OperationAdder = ({ + column, + addOperationCb, + defaultArgs, +}: { + column: string; + addOperationCb: any; + defaultArgs: any; +}): JSX.Element => { const addOperationByName = (localOperationName: string) => { return () => { const defaultOperation = defaultArgs[localOperationName]; diff --git a/js/components/RechartExtra.ts b/js/components/RechartExtra.ts new file mode 100644 index 00000000..d0a19f9c --- /dev/null +++ b/js/components/RechartExtra.ts @@ -0,0 +1,68 @@ +import _ from 'lodash'; +import { CSSProperties, ReactNode } from 'react'; +export { Global, DefaultTooltipContent } from 'recharts'; + +//export Global; + +//import { AnimationDuration, AnimationTiming } from '../util/types'; +/** The type of easing function to use for animations */ +export type AnimationTiming = + | 'ease' + | 'ease-in' + | 'ease-out' + | 'ease-in-out' + | 'linear'; +/** Specifies the duration of animation, the unit of this option is ms. */ +export type AnimationDuration = number; + +export type TooltipType = 'none'; +export type ValueType = number | string | Array; +export type NameType = number | string; + +export type Formatter = ( + value: TValue, + name: TName, + item: Payload, + index: number, + payload: Array> +) => [React.ReactNode, TName] | React.ReactNode; + +export interface Payload { + type?: TooltipType; + color?: string; + formatter?: Formatter; + name?: TName; + value?: TValue; + unit?: ReactNode; + dataKey?: string | number; + payload?: any; + chartType?: string; + stroke?: string; + strokeDasharray?: string | number; + strokeWidth?: number | string; +} + +export interface DefaultProps< + TValue extends ValueType, + TName extends NameType +> { + separator?: string; + wrapperClassName?: string; + labelClassName?: string; + formatter?: Formatter; + contentStyle?: CSSProperties; + itemStyle?: CSSProperties; + labelStyle?: CSSProperties; + labelFormatter?: ( + label: any, + payload: Array> + ) => ReactNode; + label?: any; + payload?: Array>; + itemSorter?: (item: Payload) => number | string; +} + +export const isNumber = (value: unknown): value is number => + _.isNumber(value) && !_.isNaN(value); +export const isNumOrStr = (value: unknown): value is number | string => + isNumber(value as number) || _.isString(value); diff --git a/js/components/RechartTooltip.tsx b/js/components/RechartTooltip.tsx new file mode 100644 index 00000000..bf4dc7f2 --- /dev/null +++ b/js/components/RechartTooltip.tsx @@ -0,0 +1,373 @@ +//@ts-nocheck +/** + * @fileOverview Tooltip + */ +import React, { + CSSProperties, + ReactNode, + ReactElement, + SVGProps, + useEffect, + useState, + useRef, + useCallback, +} from 'react'; +import { translateStyle } from 'react-smooth'; +import _ from 'lodash'; +//import classNames from 'classnames'; + +//import { DefaultTooltipContent, ValueType, NameType, Payload, Props as DefaultProps } from './DefaultTooltipContent'; + +import type { + ValueType, + NameType, + Payload, + DefaultProps, +} from './RechartExtra'; +//import { Global } from '../util/Global'; +//import { isNumber } from '../util/DataUtils'; +import { Global, isNumber } from './RechartExtra'; + +//import { AnimationDuration, AnimationTiming } from '../util/types'; +import { AnimationDuration, AnimationTiming } from './RechartExtra'; + +import { DefaultTooltipContent } from 'recharts'; + +const CLS_PREFIX = 'recharts-tooltip-wrapper'; + +const EPS = 1; + +export type ContentType = + | ReactElement + | ((props: TooltipProps) => ReactNode); + +type UniqueFunc = ( + entry: Payload +) => unknown; +type UniqueOption = + | boolean + | UniqueFunc; +function defaultUniqBy( + entry: Payload +) { + return entry.dataKey; +} +function getUniqPayload( + option: UniqueOption, + payload: Array> +) { + if (option === true) { + //@ts-ignore + return _.uniqBy(payload, defaultUniqBy); + } + + if (_.isFunction(option)) { + return _.uniqBy(payload, option); + } + + return payload; +} + +function renderContent( + content: ContentType, + props: TooltipProps +) { + if (React.isValidElement(content)) { + return React.cloneElement(content, props); + } + if (_.isFunction(content)) { + return React.createElement(content as any, props); + } + + return ; +} + +export type OptionalCoords = { + x?: number; + y?: number; +}; + +export type TooltipProps< + TValue extends ValueType, + TName extends NameType +> = DefaultProps & { + allowEscapeViewBox?: { + x?: boolean; + y?: boolean; + }; + reverseDirection?: { + x?: boolean; + y?: boolean; + }; + content?: ContentType; + viewBox?: { + x?: number; + y?: number; + width?: number; + height?: number; + }; + active?: boolean; + offset?: number; + wrapperStyle?: CSSProperties; + cursor?: boolean | ReactElement | SVGProps; + coordinate?: OptionalCoords; + position?: OptionalCoords; + trigger?: 'hover' | 'click'; + shared?: boolean; + payloadUniqBy?: UniqueOption; + isAnimationActive?: boolean; + animationDuration?: AnimationDuration; + animationEasing?: AnimationTiming; + filterNull?: boolean; + useTranslate3d?: boolean; + box?: OptionalCoords; +}; + +const tooltipDefaultProps: TooltipProps = { + active: false, + allowEscapeViewBox: { x: false, y: false }, + reverseDirection: { x: false, y: false }, + offset: 10, + viewBox: { x: 0, y: 0, height: 0, width: 0 }, + coordinate: { x: 0, y: 0 }, + // this doesn't exist on TooltipProps + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + cursorStyle: {}, + separator: ' : ', + wrapperStyle: {}, + contentStyle: {}, + itemStyle: {}, + labelStyle: {}, + cursor: true, + trigger: 'hover', + isAnimationActive: !Global.isSsr, + animationEasing: 'ease', + animationDuration: 400, + filterNull: true, + useTranslate3d: false, + box: { x: 0, y: 0 }, +}; + +export const Tooltip = ( + props: TooltipProps & { children?: React.ReactNode } +) => { + const [boxWidth, setBoxWidth] = useState(-1); + const [boxHeight, setBoxHeight] = useState(-1); + const [dismissed, setDismissed] = useState(false); + const [dismissedAtCoordinate, setDismissedAtCoordinate] = useState({ + x: 0, + y: 0, + }); + const [boxCoords, setBoxCoords] = useState({ x: 0, y: 0 }); + + const wrapperNode = useRef(); + const { + allowEscapeViewBox, + reverseDirection, + coordinate, + offset, + position, + viewBox, + } = props; + + const handleKeyDown = useCallback( + (event: KeyboardEvent) => { + if (event.key === 'Escape') { + setDismissed(true); + setDismissedAtCoordinate((prev) => ({ + ...prev, + x: coordinate?.x, + y: coordinate?.y, + })); + } + }, + [coordinate?.x, coordinate?.y] + ); + + useEffect(() => { + const updateBBox = () => { + if (dismissed) { + document.removeEventListener('keydown', handleKeyDown); + if ( + coordinate?.x !== dismissedAtCoordinate.x || + coordinate?.y !== dismissedAtCoordinate.y + ) { + setDismissed(false); + } + } else { + document.addEventListener('keydown', handleKeyDown); + } + + if (wrapperNode.current && wrapperNode.current.getBoundingClientRect) { + const box = wrapperNode.current.getBoundingClientRect(); + setBoxCoords({ x: box.x, y: box.y }); + if ( + Math.abs(box.width - boxWidth) > EPS || + Math.abs(box.height - boxHeight) > EPS + ) { + setBoxWidth(box.width); + setBoxHeight(box.height); + } + } else if (boxWidth !== -1 || boxHeight !== -1) { + setBoxWidth(-1); + setBoxHeight(-1); + } + }; + + updateBBox(); + + return () => { + document.removeEventListener('keydown', handleKeyDown); + }; + }, [ + boxHeight, + boxWidth, + coordinate, + dismissed, + dismissedAtCoordinate.x, + dismissedAtCoordinate.y, + handleKeyDown, + ]); + + const getTranslate = ({ + key, + tooltipDimension, + viewBoxDimension, + }: { + key: 'x' | 'y'; + tooltipDimension: number; + viewBoxDimension: number; + }) => { + if (position && isNumber(position[key])) { + return position[key]; + } + + const pCoordinate = coordinate || { x: 0, y: 0 }; + const negative = (pCoordinate[key] || 0) - tooltipDimension - (offset || 0); + const positive = (pCoordinate[key] || 0) + offset; + if (allowEscapeViewBox?.[key]) { + return reverseDirection[key] ? negative : positive; + } + + if (reverseDirection?.[key]) { + const tooltipBoundary = negative; + const viewBoxBoundary = viewBox[key]; + if (tooltipBoundary < viewBoxBoundary) { + return Math.max(positive, viewBox[key]); + } + return Math.max(negative, viewBox[key]); + } + const tooltipBoundary = positive + tooltipDimension; + const viewBoxBoundary = viewBox[key] + viewBoxDimension; + if (tooltipBoundary > viewBoxBoundary) { + return Math.max(negative, viewBox[key]); + } + return Math.max(positive, viewBox[key]); + }; + + const { + payload, + payloadUniqBy, + filterNull, + active, + wrapperStyle, + useTranslate3d, + isAnimationActive, + animationDuration, + animationEasing, + } = props; + const finalPayload = getUniqPayload( + payloadUniqBy, + filterNull && payload && payload.length + ? payload.filter((entry) => !_.isNil(entry.value)) + : payload + ); + const hasPayload = finalPayload && finalPayload.length; + const { content } = props; + let outerStyle: CSSProperties = { + pointerEvents: 'none', + visibility: !dismissed && active && hasPayload ? 'visible' : 'hidden', + position: 'absolute', + top: 0, + left: 0, + ...wrapperStyle, + }; + let translateX, translateY; + + if (position && isNumber(position.x) && isNumber(position.y)) { + translateX = position.x; + translateY = position.y; + } else if (boxWidth > 0 && boxHeight > 0 && coordinate) { + translateX = getTranslate({ + key: 'x', + tooltipDimension: boxWidth, + viewBoxDimension: viewBox.width, + }); + + translateY = getTranslate({ + key: 'y', + tooltipDimension: boxHeight, + viewBoxDimension: viewBox.height, + }); + } else { + outerStyle.visibility = 'hidden'; + } + + outerStyle = { + ...translateStyle({ + transform: useTranslate3d + ? `translate3d(${translateX}px, ${translateY}px, 0)` + : `translate(${translateX}px, ${translateY}px)`, + }), + ...outerStyle, + }; + + if (isAnimationActive && active) { + outerStyle = { + ...translateStyle({ + transition: `transform ${animationDuration}ms ${animationEasing}`, + }), + ...outerStyle, + }; + } + + // eslint-disable-next-line + // const cls = classNames(CLS_PREFIX, { + // [`${CLS_PREFIX}-right`]: isNumber(translateX) && coordinate && isNumber(coordinate.x) && translateX >= coordinate.x, + // [`${CLS_PREFIX}-left`]: isNumber(translateX) && coordinate && isNumber(coordinate.x) && translateX < coordinate.x, + // [`${CLS_PREFIX}-bottom`]: + // isNumber(translateY) && coordinate && isNumber(coordinate.y) && translateY >= coordinate.y, + // [`${CLS_PREFIX}-top`]: isNumber(translateY) && coordinate && isNumber(coordinate.y) && translateY < coordinate.y, + // }); + + const cls = `${CLS_PREFIX}-top`; + return ( + // ESLint is disabled to allow listening to the `Escape` key. Refer to + // https://github.com/recharts/recharts/pull/2925 + // eslint-disable-next-line jsx-a11y/no-noninteractive-element-interactions +
+ {renderContent(content, { + ...props, + box: boxCoords, + payload: finalPayload, + })} +
+ ); +}; + +// needs to be set so that renderByOrder can find the correct handler function +Tooltip.displayName = 'Tooltip'; + +/** + * needs to be set so that renderByOrder can access an have default values for + * children.props when there are no props set by the consumer + * doesn't work if using default parameters + */ +Tooltip.defaultProps = tooltipDefaultProps; diff --git a/js/components/StatusBar.tsx b/js/components/StatusBar.tsx index 991f17fb..fdce7431 100644 --- a/js/components/StatusBar.tsx +++ b/js/components/StatusBar.tsx @@ -1,13 +1,8 @@ -import React, { - useState, - useRef, -} from 'react'; +// https://plnkr.co/edit/QTNwBb2VEn81lf4t?open=index.tsx +import React, { useState, useRef } from 'react'; import _ from 'lodash'; import { AgGridReact } from 'ag-grid-react'; // the AG Grid React Component -import { - ColDef, - GridOptions, -} from 'ag-grid-community'; +import { ColDef, GridOptions } from 'ag-grid-community'; import { intFormatter } from './gridUtils'; export type setColumFunc = (newCol: string) => void; @@ -18,48 +13,61 @@ export interface DfConfig { sampleSize: number; sampled: boolean; summaryStats: boolean; - showCommands:boolean; + showCommands: boolean; //reorderdColumns: boolean; } - const columnDefs: ColDef[] = [ - { field: 'summaryStats', - headerName:'Σ', - headerTooltip:'Summary Stats', - width:30 + { + field: 'summaryStats', + headerName: 'Σ', + headerTooltip: 'Summary Stats', + width: 30, + }, + // { field: 'reorderdColumns', + // headerName: "Θ", + // headerTooltip:"Reorder Columns", + // width:30 + // }, + { + field: 'showCommands', + headerName: 'λ', + headerTooltip: 'Show Commands', + width: 30, }, -// { field: 'reorderdColumns', -// headerName: "Θ", -// headerTooltip:"Reorder Columns", -// width:30 -// }, - { field: 'showCommands', - headerName: "λ", - headerTooltip:"Show Commands", - width:30 -}, - { field: 'sampled', - headerName: "Ξ", - headerTooltip:"Sampled", - width:30 -}, - { field: 'help', - headerName: "?", - headerTooltip:"Help", - width:30, - cellRenderer: function(params:any) { - return ?} -}, + { field: 'sampled', headerName: 'Ξ', headerTooltip: 'Sampled', width: 30 }, + { + field: 'help', + headerName: '?', + headerTooltip: 'Help', + width: 30, + cellRenderer: function (params: any) { + return ( + + ? + + ); + }, + }, - { field: 'totalRows', width:100}, - { field: 'columns', width:100 }, - { field: 'rowsShown', width:120}, - { field: 'sampleSize', width:120 } + { field: 'totalRows', width: 100 }, + { field: 'columns', width: 100 }, + { field: 'rowsShown', width: 120 }, + { field: 'sampleSize', width: 120 }, ]; -export function StatusBar({ config, setConfig }: { config:any, setConfig:any }) { +export function StatusBar({ + config, + setConfig, +}: { + config: any; + setConfig: any; +}) { const { totalRows, columns, @@ -75,28 +83,26 @@ export function StatusBar({ config, setConfig }: { config:any, setConfig:any }) { totalRows: intFormatter.format(totalRows), columns, - rowsShown : intFormatter.format(rowsShown), - sampleSize : intFormatter.format(sampleSize), - sampled: sampled ? "1" : "0", - summaryStats: summaryStats ? "1" : "0", + rowsShown: intFormatter.format(rowsShown), + sampleSize: intFormatter.format(sampleSize), + sampled: sampled ? '1' : '0', + summaryStats: summaryStats ? '1' : '0', // reorderdColumns: reorderdColumns ? "Ϋ" : "ό", - showCommands: showCommands ? "1" : "0" + showCommands: showCommands ? '1' : '0', }, ]; - const updateDict = (event:any) => { + const updateDict = (event: any) => { const colName = event.column.getColId(); if (colName === 'summaryStats') { setConfig({ ...config, summaryStats: !config.summaryStats }); - } - else if (colName === 'sampled') { + } else if (colName === 'sampled') { setConfig({ ...config, sampled: !config.sampled }); } else if (colName === 'showCommands') { setConfig({ ...config, showCommands: !config.showCommands }); } // } else if (colName === 'reorderdColumns') { // setConfig({ ...config, reorderdColumns: !config.reorderdColumns }); - }; const gridOptions: GridOptions = { rowSelection: 'single', diff --git a/js/components/gridUtils.ts b/js/components/gridUtils.ts index ea26f34d..b2ae809d 100644 --- a/js/components/gridUtils.ts +++ b/js/components/gridUtils.ts @@ -1,4 +1,8 @@ -import { ColDef, ValueFormatterFunc, ValueFormatterParams } from 'ag-grid-community'; +import { + ColDef, + ValueFormatterFunc, + ValueFormatterParams, +} from 'ag-grid-community'; import { DFWhole, DFColumn, ColumnHint } from './staticData'; import _ from 'lodash'; export const updateAtMatch = ( @@ -17,33 +21,37 @@ export const updateAtMatch = ( return retColumns; }; -export const intFormatter = new Intl.NumberFormat( - 'en-US', { minimumFractionDigits:0, maximumFractionDigits: 3 }); +export const intFormatter = new Intl.NumberFormat('en-US', { + minimumFractionDigits: 0, + maximumFractionDigits: 3, +}); -const floatFormatter = new Intl.NumberFormat('en-US', { minimumFractionDigits:3, maximumFractionDigits: 3 }); +const floatFormatter = new Intl.NumberFormat('en-US', { + minimumFractionDigits: 3, + maximumFractionDigits: 3, +}); -export const anyFormatter = (params: ValueFormatterParams):string => { +export const anyFormatter = (params: ValueFormatterParams): string => { const val = params.value; try { - const num = Number(val) + const num = Number(val); if (val === null) { - return ""; + return ''; } else if (val === undefined) { - return ""; - } - else if (Number.isFinite(num)) { + return ''; + } else if (Number.isFinite(num)) { if (Number.isInteger(num)) { - const formatted = intFormatter.format(num) - return `${formatted} ` + const formatted = intFormatter.format(num); + return `${formatted} `; } else { - return floatFormatter.format(num) + return floatFormatter.format(num); } } - } catch (e:any) { + } catch (e: any) { //console.log("formatting error", val, e); } - return val -} + return val; +}; /* console.log((new Intl.NumberFormat('en-US')).format(amount)) console.log((new Intl.NumberFormat('en-US', { maximumFractionDigits: 1})).format(number)) @@ -57,7 +65,7 @@ export const anyFormatter = (params: ValueFormatterParams):string => { return '£' + formatNumber(params.value); } */ -function getFormatter(hint: ColumnHint): ValueFormatterFunc { +function getFormatter(hint: ColumnHint): ValueFormatterFunc { if (hint === undefined || hint.is_numeric === false) { return anyFormatter; } else { @@ -65,8 +73,8 @@ function getFormatter(hint: ColumnHint): ValueFormatterFunc { if (hint.is_integer) { const totalWidth = commas + hint.max_digits; - return (params: ValueFormatterParams):string => { - console.log("params", params) + return (params: ValueFormatterParams): string => { + console.log('params', params); const formatter = new Intl.NumberFormat('en-US'); return formatter.format(params.value).padStart(totalWidth, ' '); @@ -88,12 +96,11 @@ function getFormatter(hint: ColumnHint): ValueFormatterFunc { return numFormatted.padStart(intWidth, " ").padEnd(intWidth + fracWidth, " ") } };*/ - return (params: ValueFormatterParams):string => { + return (params: ValueFormatterParams): string => { //console.log("params", params) return floatFormatter.format(params.value); }; - } } } @@ -102,13 +109,18 @@ export function dfToAgrid(tdf: DFWhole): [ColDef[], unknown[]] { const fields = tdf.schema.fields; //console.log("tdf", tdf); //console.log("hints", tdf.table_hints); - const retColumns:ColDef[] = fields.map((f: DFColumn) => { + const retColumns: ColDef[] = fields.map((f: DFColumn) => { //console.log(f.name, tdf.table_hints[f.name]) - const colDef:ColDef = {field: f.name, valueFormatter:getFormatter(_.get(tdf.table_hints,f.name, { is_numeric:false})) } + const colDef: ColDef = { + field: f.name, + valueFormatter: getFormatter( + _.get(tdf.table_hints, f.name, { is_numeric: false }) + ), + }; if (f.name === 'index') { colDef.pinned = 'left'; } - return colDef; + return colDef; }); return [retColumns, tdf.data]; } diff --git a/js/components/staticData.ts b/js/components/staticData.ts index 1c0418f1..26822d07 100644 --- a/js/components/staticData.ts +++ b/js/components/staticData.ts @@ -30,48 +30,151 @@ const origRows = [ ]; */ export interface DFColumn { - name: string; type:string; + name: string; + type: string; } export type DFDataRow = Record; export type DFData = DFDataRow[]; export interface ColumnObjHint { - is_numeric:false; + is_numeric: false; + histogram?: any[]; } export interface ColumnNumHint { - is_numeric:true; - is_integer:boolean; - min_digits:number; - max_digits:number; - histogram:number[][]; + is_numeric: true; + is_integer: boolean; + min_digits: number; + max_digits: number; + histogram: any[]; } -export type ColumnHint = ColumnObjHint | ColumnNumHint +export type ColumnHint = ColumnObjHint | ColumnNumHint; export interface DFWhole { schema: { fields: DFColumn[]; primaryKey: string[]; //['index'] pandas_version: string; //'1.4.0', - - }; - table_hints:Record; + table_hints: Record; data: DFData; } export const EmptyDf: DFWhole = { - schema: { fields: [], primaryKey:[], pandas_version:"" }, - table_hints:{}, + schema: { fields: [], primaryKey: [], pandas_version: '' }, + table_hints: {}, data: [], }; //print(sdf.to_json(orient='table', indent=2)) -//export const tableDf2:DFWhole = { -export const foo:DFWhole = { +export const histograms = { + num_histo : [ + {'name': '-406 - -332', 'population': 1}, + {'name': '-332 - -258', 'population': 0}, + {'name': '-258 - -184', 'population': 2}, + {'name': '-184 - -111', 'population': 10}, + {'name': '-111 - -37', 'population': 22}, + {'name': '-37 - 36', 'population': 30}, + {'name': '36 - 109', 'population': 22}, + {'name': '109 - 183', 'population': 10}, + {'name': '183 - 257', 'population': 3}, + {'name': '257 - 331', 'population': 0}], + + bool_histo : [ + {'name': 'False', 'false': 50}, + {'name': 'True', 'true': 30}, + {'name': 'NA', 'NA': 20}], + + NA_Only : [ + {'name': 'NA', 'NA': 100}], + + simple_catgeorical : [ + {'name': 2, 'cat_pop': 87.0}, + {'name': 1, 'cat_pop': 13.0}], + + categorical_histo : [ + {'name': 'KTM', 'cat_pop': 30}, + {'name': 'Gas Gas', 'cat_pop': 15}, + {'name': 'Yamaha', 'cat_pop': 10}, + {'name': 'unique', 'unique': 25}, + {'name': 'NA', 'NA': 20} + ], + + categorical_histo_lt : [ + {'name': 'KTM', 'cat_pop': 25}, + {'name': 'Gas Gas', 'cat_pop': 12}, + {'name': 'Yamaha', 'cat_pop': 8}, + {'name': 'NA', 'NA': 20}, + {'name': 'longtail', 'unique': 15, 'longtail':20}, + ], + + all_unique : [ + {'name': 'unique', 'unique': 100}, + ], + + unique_na : [ + {'name': 'unique', 'unique': 80}, + {'name': 'NA', 'NA': 20} + ], + unique_continuous : [ + {'name': '-406 -332', 'population': 1}, + {'name': '-332 -258', 'population': 0}, + {'name': '-258 -184', 'population': 0}, + {'name': '-184 -111', 'population': 10}, + {'name': '-111 -37', 'population': 21}, + {'name': '-37 36', 'population': 29}, + {'name': '36 109', 'population': 22}, + {'name': '109 183', 'population': 9}, + {'name': '183 257', 'population': 3}, + {'name': '257 331', 'population': 0}, + {'name': 'unique', 'unique': 100}, + ], + + unique_continuous_scaled : [ + {'name': '-406 -332', 'population': 0}, + {'name': '-332 -258', 'population': 0}, + {'name': '-258 -184', 'population': 0}, + {'name': '-184 -111', 'population': 10}, + {'name': '-111 -37', 'population': 21}, + {'name': '-37 36', 'population': 29}, + {'name': '36 109', 'population': 22}, + {'name': '109 183', 'population': 9}, + {'name': '183 257', 'population': 3}, + {'name': '257 331', 'population': 0}, + {'name': 'unique', 'unique': 29}, + ], + + unique_continuous_scaled_50 : [ + {'name': '-406 -332', 'population': 0}, + {'name': '-332 -258', 'population': 0}, + {'name': '-258 -184', 'population': 0}, + {'name': '-184 -111', 'population': 10}, + {'name': '-111 -37', 'population': 21}, + {'name': '-37 36', 'population': 29}, + {'name': '36 109', 'population': 22}, + {'name': '109 183', 'population': 9}, + {'name': '183 257', 'population': 3}, + {'name': '257 331', 'population': 0}, + {'name': 'longtail', 'unique': 15}, + ], + start_station_categorical : [{'name': 'Pershing Square N', 'cat_pop': 1}, + {'name': '8 Ave & W 31 St', 'cat_pop': 1}, + {'name': 'Lafayette St & E 8 St', 'cat_pop': 1}, + {'name': 'W 21 St & 6 Ave', 'cat_pop': 1}, + {'name': 'E 17 St & Broadway', 'cat_pop': 1}, + {'name': '8 Ave & W 33 St', 'cat_pop': 1}, + {'name': 'E 43 St & Vanderbilt Ave', 'cat_pop': 1}, + {'name': 'unique', 'cat_pop': 0}, + {'name': 'long_tail', 'cat_pop': 92}, + {'name': 'NA', 'cat_pop': 0}] + +}; + +//export const tableDf2:DFWhole = { +export const foo: DFWhole = { schema: { fields: [ { name: 'index', type: 'integer' }, @@ -89,16 +192,18 @@ export const foo:DFWhole = { primaryKey: ['index'], pandas_version: '1.4.0', }, - table_hints: { 'index': { is_numeric:false}, - 'tripduration': { is_numeric:false}, - 'starttime': { is_numeric:false}, - 'stoptime': { is_numeric:false}, - 'start station id': { is_numeric:false}, - 'start station name': { is_numeric:false}, - 'start station latitude': { is_numeric:false}, - 'bikeid': { is_numeric:false}, - 'birth year': { is_numeric:false}, - 'gender': { is_numeric:false}}, + table_hints: { + index: { is_numeric: false }, + tripduration: { is_numeric: false, histogram:histograms.num_histo}, + starttime: { is_numeric: false }, + stoptime: { is_numeric: false }, + 'start station id': { is_numeric: false }, + 'start station name': { is_numeric: false }, + 'start station latitude': { is_numeric: false }, + bikeid: { is_numeric: false }, + 'birth year': { is_numeric: false }, + gender: { is_numeric: false }, + }, data: [ { index: 0, @@ -164,63 +269,81 @@ export const foo:DFWhole = { }; export const tableDf: DFWhole = { - 'schema': {'fields': [ - {'name': 'index', 'type': 'integer'}, - {'name': 'end station name', 'type': 'string'}, - {'name': 'tripduration', 'type': 'integer'}, - {'name': 'start station name', 'type': 'string'}, - {'name': 'floatCol', 'type': 'float'} + schema: { + fields: [ + { name: 'index', type: 'integer' }, + { name: 'end station name', type: 'string' }, + { name: 'tripduration', type: 'integer' }, + { name: 'start station name', type: 'string' }, + { name: 'floatCol', type: 'float' }, + ], + primaryKey: ['index'], + pandas_version: '1.4.0', + }, + data: [ + { + index: 0, + 'end station name': 'Elizabeth St & Hester St', + tripduration: 471, + 'start station name': 'Catherine St & Monroe St', + floatCol: '1.111', + }, + { + index: 1, + 'end station name': 'South St & Whitehall St', + tripduration: 1494, + 'start station name': '1 Ave & E 30 St', + floatCol: '8.888', + }, + { + index: 2, + 'end station name': 'E 59 St & Sutton Pl', + tripduration: 464, + 'start station name': 'E 48 St & 3 Ave', + floatCol: '9.999', + }, + { + index: 3, + 'end station name': 'E 33 St & 5 Ave', + tripduration: 373, + 'start station name': 'Pershing Square N', + floatCol: '-10.1', + }, + { + index: 4, + 'end station name': 'Hancock St & Bedford Ave', + tripduration: 660, + 'start station name': 'Atlantic Ave & Fort Greene Pl', + floatCol: '10.99', + }, ], - 'primaryKey': ['index'], - 'pandas_version': '1.4.0'}, - 'data': [{'index': 0, - 'end station name': 'Elizabeth St & Hester St', - 'tripduration': 471, - 'start station name': 'Catherine St & Monroe St', - 'floatCol': '1.111', - } - - , - {'index': 1, - 'end station name': 'South St & Whitehall St', - 'tripduration': 1494, - 'start station name': '1 Ave & E 30 St', - 'floatCol': '8.888' - }, - {'index': 2, - 'end station name': 'E 59 St & Sutton Pl', - 'tripduration': 464, - 'start station name': 'E 48 St & 3 Ave', - 'floatCol': '9.999' - }, - {'index': 3, - 'end station name': 'E 33 St & 5 Ave', - 'tripduration': 373, - 'start station name': 'Pershing Square N', - 'floatCol': '-10.1' - }, - {'index': 4, - 'end station name': 'Hancock St & Bedford Ave', - 'tripduration': 660, - 'start station name': 'Atlantic Ave & Fort Greene Pl', - 'floatCol': '10.99' - }], - 'table_hints': - { - 'end station name': {'is_numeric': false}, - 'tripduration': {'is_numeric': true, - 'is_integer': true, - 'min_digits': 3, - 'max_digits': 4, - 'histogram': [[0.6, 0.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2], - [373.0, 485.1, 597.2, 709.3, 821.4, 933.5, 1045.6, - 1157.7, 1269.8, 1381.9, 1494.0]]}, - 'start station name': {'is_numeric': false}, - 'floatCol': {'is_numeric': true, - 'is_integer': false, - 'min_digits': 1, - 'max_digits': 3, - 'histogram': [[0.6, 0.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2], - [373.0, 485.1, 597.2, 709.3, 821.4, 933.5, 1045.6, - 1157.7, 1269.8, 1381.9, 1494.0]]}, - }} + table_hints: { + 'end station name': { is_numeric: false, histogram: histograms.categorical_histo_lt }, + tripduration: { + is_numeric: true, + is_integer: true, + min_digits: 3, + max_digits: 4, + histogram:histograms.num_histo + }, + 'start station name': { is_numeric: false, histogram: histograms.bool_histo }, + floatCol: { + is_numeric: true, + is_integer: false, + min_digits: 1, + max_digits: 3, + histogram: [ + { name: 521, cat_pop: 0.0103 }, + { name: 358, cat_pop: 0.0096 }, + { name: 519, cat_pop: 0.009 }, + { name: 497, cat_pop: 0.0087 }, + { name: 293, cat_pop: 0.0082 }, + { name: 285, cat_pop: 0.0081 }, + { name: 435, cat_pop: 0.008 }, + { name: 'unique', cat_pop: 0.0001 }, + { name: 'long_tail', cat_pop: 0.938 }, + { name: 'NA', cat_pop: 0.0 }, + ], + }, + }, +}; diff --git a/js/dcefwidget.ts b/js/dcefwidget.ts index 5d8f1920..65f4ccf3 100644 --- a/js/dcefwidget.ts +++ b/js/dcefwidget.ts @@ -77,7 +77,7 @@ export class DCEFWidgetView extends DOMWidgetView { } return React.createElement(WidgetDCFCell, props); }; - console.log("widget el", this.el) + console.log('widget el', this.el); /* const el = this.el; diff --git a/js/style/dcf-npm.css b/js/style/dcf-npm.css index cd62d846..be5556c9 100644 --- a/js/style/dcf-npm.css +++ b/js/style/dcf-npm.css @@ -163,6 +163,53 @@ div.dependent-tabs ul.tabs li.active { width:800px; border:3px dashed purple; + + + +} + +.custom-tooltip { + color:black; + background:green; +} + + +.histogram-component { + margin:0; + border-left:1px solid #68686e; + border-right:1px solid #68686e; + + +} + +.floating-tooltip { + color:black;; + background:white; } + +.floating-tooltip dl { + border:1px solid black; + padding:1px 4px; + margin:0; + display:flow-root; +} +.floating-tooltip dl dt { + font-weight:bold; + float:left; + margin:0; + padding:0; + +} +.floating-tooltip dl dd { + + float:left; + margin:0 0 0 10px; + clear:right; +} + + + /*# sourceMappingURL=dcf-npm.css.map */ + + diff --git a/package.json b/package.json index 4e7344bb..96f6d06c 100644 --- a/package.json +++ b/package.json @@ -1,174 +1,177 @@ { - "name": "buckaroo", - "version": "0.2.26", - "description": "Fast Datagrid widget for the Jupyter Notebook and JupyterLab", - "keywords": [ - "jupyter", - "jupyterlab", - "jupyterlab-extension", - "widgets" - ], - "files": [ - "lib/**/*.js", - "lib/**/*.js.map", - "lib/**/*.ts", - "dist/*.js", - "dist/*.js.map", - "dist/*.png", - "style/**/*.*" - ], - "homepage": "https://github.com/paddymul/buckaroo", - "bugs": { - "url": "https://github.com/paddymul/buckaroo/issues" - }, - "license": "BSD-3-Clause", - "author": { - "name": "Paddy Mullen", - "email": "" - }, - "main": "lib/index.js", - "types": "./lib/index.d.ts", - "repository": { - "type": "git", - "url": "https://github.com/paddymul/buckaroo" - }, - "scripts": { - "build": "npm run build:lib && npm run build:nbextension && npm run build:labextension", - "build-full": "npm run build:lib && run build:nbextension && npm run build:labextension", - "build:dev": "npm run build:lib && npm run build:nbextension && npm run build:labextension:dev", - "build:labextension": "jupyter labextension build .", - "build:labextension:dev": "which jupyter && jupyter labextension build --development True .", - "build:lib": "tsc", - "build:nbextension": "webpack --mode=production --no-devtool", - "build:widget-examples": "cd widget-examples/basic && webpack --mode=production", - "build:all": "npm run build:labextension && npm run build:nbextension && npm run build:widget-examples", - "clean": "rimraf dist && npm run clean:lib && npm run clean:labextension && npm run clean:nbextension", - "clean:lib": "rimraf lib", - "clean:labextension": "rimraf buckaroo/labextension", - "clean:nbextension": "rimraf buckaroo/nbextension/index.*", - "lint": "eslint 'js/**/*.{ts,tsx}'", - "lint:check": "eslint 'js/**/*.{ts,tsx}'", - "lint:fix": "eslint 'js/**/*.{ts,tsx}' --fix", - "prepack": "npm run build:labextension && npm run build:nbextension", - "test": "jest --verbose --passWithNoTests", - "watch": "npm-run-all -p watch:*", - "watch:lib": "tsc -w", - "watch:nbextension": "webpack --watch", - "watch:labextension": "jupyter labextension watch .", - "dev": "webpack-cli serve --mode=development --env development --open" - }, - "dependencies": { - "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^6.0.0", - "@jupyterlab/apputils": "^3.0.2", - "ag-grid-community": "^29.2.0", - "ag-grid-react": "^29.2.0", - "lodash": "^4.17.21", - "react": "^18.0.0", - "react-dom": "^18.0.0" - }, - "devDependencies": { - "@babel/cli": "^7.6.3", - "@babel/core": "^7.6.3", - "@babel/preset-env": "^7.6.11", - "@babel/preset-typescript": "7.16.7", - "@jupyterlab/builder": "^3.0.1", - "@types/jest": "^28", - "@types/node": "^10.11.6", - "@types/react": "^18.0.0", - "@types/react-dom": "^18.0.0", - "@types/webpack-dev-server": "^3.11.1", - "@types/webpack-env": "^1.13.6", - "@typescript-eslint/eslint-plugin": "^5.9.1", - "@typescript-eslint/parser": "^5.9.1", - "acorn": "^6.2.0", - "babel-jest": "^28", - "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2", - "css-loader": "^5.0.0", - "eslint": "^8.6.0", - "eslint-config-prettier": "^8.3.0", - "eslint-plugin-prettier": "^4.0.0", - "eslint-plugin-react": "^7.28.0", - "eslint-plugin-react-hooks": "^4.3.0", - "fs-extra": "^7.0.0", - "html-loader": "^2.1.2", - "html-webpack-plugin": "^5.0.0", - "jest": "^28", - "lint-staged": "^10.2.11", - "markdown-loader": "^7.0.0", - "mkdirp": "^0.5.1", - "npm-run-all": "^4.1.3", - "prettier": "^2.6.2", - "prettier-standalone": "^1.3.1-0", - "prismjs": "^1.28.0", - "postcss": "^8.4.14", - "postcss-loader": "^7.0.1", - "postcss-preset-env": "^7.7.2", - "postcss-nested": "^6.0.0", - "sass": "^1.53.0", - "sass-loader": "^13.0.2", - "react-router": "^6.3.0", - "react-router-dom": "^5.2.0", - "rimraf": "^3.0.2", - "source-map-loader": "^0.2.4", - "style-loader": "^2.0.0", - "svg-url-loader": "^7.1.1", - "ts-jest": "^28.0.8", - "ts-loader": "^8.0.14", - "ts-node": "^9.1.1", - "tsconfig-paths-webpack-plugin": "^3.3.0", - "typescript": "^4.4.3", - "url-loader": "^4.1.0", - "webpack": "^5", - "webpack-cli": "^4.5.0", - "webpack-dev-server": "^4.0.0" - }, - "jupyterlab": { - "extension": "lib/plugin", - "webpackConfig": "webpack.lab.config.js", - "outputDir": "./buckaroo/labextension", - "sharedPackages": { - "@jupyter-widgets/base": { - "bundled": false, - "singleton": true - }, - "@lumino/algorithm": { - "bundled": false, - "singleton": true - }, - "@lumino/commands": { - "bundled": false, - "singleton": true - }, - "@lumino/coreutils": { - "bundled": false, - "singleton": true - }, - "@lumino/datagrid": { - "bundled": false, - "singleton": true - }, - "@lumino/default-theme": { - "bundled": false, - "singleton": true - }, - "@lumino/domutils": { - "bundled": false, - "singleton": true - }, - "@lumino/messaging": { - "bundled": false, - "singleton": true - }, - "@lumino/virtualdom": { - "bundled": false, - "singleton": true - }, - "@lumino/widgets": { - "bundled": false, - "singleton": true - }, - "react": false, - "react-dom": false - } + "name": "buckaroo", + "version": "0.2.27", + "description": "Fast Datagrid widget for the Jupyter Notebook and JupyterLab", + "keywords": [ + "jupyter", + "jupyterlab", + "jupyterlab-extension", + "widgets" + ], + "files": [ + "lib/**/*.js", + "lib/**/*.js.map", + "lib/**/*.ts", + "dist/*.js", + "dist/*.js.map", + "dist/*.png", + "style/**/*.*" + ], + "homepage": "https://github.com/paddymul/buckaroo", + "bugs": { + "url": "https://github.com/paddymul/buckaroo/issues" + }, + "license": "BSD-3-Clause", + "author": { + "name": "Paddy Mullen", + "email": "" + }, + "main": "lib/index.js", + "types": "./lib/index.d.ts", + "repository": { + "type": "git", + "url": "https://github.com/paddymul/buckaroo" + }, + "scripts": { + "build": "npm run build:lib && npm run build:nbextension && npm run build:labextension", + "build-full": "npm run build:lib && run build:nbextension && npm run build:labextension", + "build:dev": "npm run build:lib && npm run build:nbextension && npm run build:labextension:dev", + "build:labextension": "jupyter labextension build .", + "build:labextension:dev": "which jupyter && jupyter labextension build --development True .", + "build:lib": "tsc", + "build:nbextension": "webpack --mode=production --no-devtool", + "build:widget-examples": "cd widget-examples/basic && webpack --mode=production", + "build:all": "npm run build:labextension && npm run build:nbextension && npm run build:widget-examples", + "clean": "rimraf dist && npm run clean:lib && npm run clean:labextension && npm run clean:nbextension", + "clean:lib": "rimraf lib", + "clean:labextension": "rimraf buckaroo/labextension", + "clean:nbextension": "rimraf buckaroo/nbextension/index.*", + "lint": "eslint 'js/**/*.{ts,tsx}'", + "lint:check": "eslint 'js/**/*.{ts,tsx}'", + "lint:fix": "eslint 'js/**/*.{ts,tsx}' --fix", + "prepack": "npm run build:labextension && npm run build:nbextension", + "test": "jest --verbose --passWithNoTests", + "watch": "npm-run-all -p watch:*", + "watch:lib": "tsc -w", + "watch:nbextension": "webpack --watch", + "watch:labextension": "jupyter labextension watch .", + "dev": "webpack-cli serve --mode=development --env development --open" + }, + "dependencies": { + "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^6.0.0", + "@jupyterlab/apputils": "^3.0.2", + "ag-grid-community": "^29.2.0", + "ag-grid-react": "^29.2.0", + "lodash": "^4.17.21", + "react": "^18.0.0", + "react-dom": "^18.0.0", + "react-smooth": "^2.0.3", + "recharts": "^2.7.3" + }, + "devDependencies": { + "@babel/cli": "^7.6.3", + "@babel/core": "^7.6.3", + "@babel/preset-env": "^7.6.11", + "@babel/preset-typescript": "7.16.7", + "@jupyterlab/builder": "^3.0.1", + "@types/jest": "^28", + "@types/node": "^10.11.6", + "@types/react": "^18.0.0", + "@types/react-dom": "^18.0.0", + "@types/webpack-dev-server": "^3.11.1", + "@types/webpack-env": "^1.13.6", + "@typescript-eslint/eslint-plugin": "^5.9.1", + "@typescript-eslint/parser": "^5.9.1", + "acorn": "^6.2.0", + "babel-jest": "^28", + "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2", + "bootstrap": "^5.1.3", + "css-loader": "^5.0.0", + "eslint": "^8.6.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-prettier": "^4.0.0", + "eslint-plugin-react": "^7.28.0", + "eslint-plugin-react-hooks": "^4.3.0", + "fs-extra": "^7.0.0", + "html-loader": "^2.1.2", + "html-webpack-plugin": "^5.0.0", + "jest": "^28", + "lint-staged": "^10.2.11", + "markdown-loader": "^7.0.0", + "mkdirp": "^0.5.1", + "npm-run-all": "^4.1.3", + "prettier": "^2.6.2", + "prettier-standalone": "^1.3.1-0", + "prismjs": "^1.28.0", + "postcss": "^8.4.14", + "postcss-loader": "^7.0.1", + "postcss-preset-env": "^7.7.2", + "postcss-nested": "^6.0.0", + "sass": "^1.53.0", + "sass-loader": "^13.0.2", + "react-router": "^6.3.0", + "react-router-dom": "^5.2.0", + "rimraf": "^3.0.2", + "source-map-loader": "^0.2.4", + "style-loader": "^2.0.0", + "svg-url-loader": "^7.1.1", + "ts-jest": "^28.0.8", + "ts-loader": "^8.0.14", + "ts-node": "^9.1.1", + "tsconfig-paths-webpack-plugin": "^3.3.0", + "typescript": "^4.4.3", + "url-loader": "^4.1.0", + "webpack": "^5", + "webpack-cli": "^4.5.0", + "webpack-dev-server": "^4.0.0" + }, + "jupyterlab": { + "extension": "lib/plugin", + "webpackConfig": "webpack.lab.config.js", + "outputDir": "./buckaroo/labextension", + "sharedPackages": { + "@jupyter-widgets/base": { + "bundled": false, + "singleton": true + }, + "@lumino/algorithm": { + "bundled": false, + "singleton": true + }, + "@lumino/commands": { + "bundled": false, + "singleton": true + }, + "@lumino/coreutils": { + "bundled": false, + "singleton": true + }, + "@lumino/datagrid": { + "bundled": false, + "singleton": true + }, + "@lumino/default-theme": { + "bundled": false, + "singleton": true + }, + "@lumino/domutils": { + "bundled": false, + "singleton": true + }, + "@lumino/messaging": { + "bundled": false, + "singleton": true + }, + "@lumino/virtualdom": { + "bundled": false, + "singleton": true + }, + "@lumino/widgets": { + "bundled": false, + "singleton": true + }, + "react": false, + "react-dom": false } + } } diff --git a/pyproject.toml b/pyproject.toml index c614f8da..0926b1cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "ipywidgets>=7.6.0,<9", "graphlib_backport>=1.0.0" ] -version = "0.3.18" +version = "0.3.19" [project.license] file = "LICENSE.txt" diff --git a/tests/auto_clean_test.py b/tests/auto_clean_test.py new file mode 100644 index 00000000..a4c9c2a0 --- /dev/null +++ b/tests/auto_clean_test.py @@ -0,0 +1,130 @@ +import unittest +import pytest + +import pandas as pd +from buckaroo import auto_clean as ac +from pandas import NA as NA +from numpy import nan +from pandas.testing import assert_series_equal, assert_frame_equal + + +DATETIME_META = {'datetime': 0.75, 'datetime_error': 0.25, 'int': 0.25, 'int_error': 0.75, 'float': 0.25, 'float_error': 0.75, 'bool': 0, 'bool_error':1} + +INT_META = {'datetime': 0.0, 'datetime_error': 1.0, 'int': 0.75, 'int_error': 0.25, 'float': 0.75, 'float_error': 0.25, 'bool': 0.0, 'bool_error': 1.0} + +FLOAT_META = {'datetime': 0.0, 'datetime_error': 1.0, 'int': 0.25, 'int_error': 0.75, 'float': 0.75, 'float_error': 0.25, 'bool': 0.0, 'bool_error': 1.0} + +STRING_META = {'datetime': 0.0, 'datetime_error': 1.0, 'int': 0.0, 'int_error': 1.0, 'float': 0.25, 'float_error': 0.75, 'bool': 0.0, 'bool_error': 1.0} + +DATETIME_EDGECASE_META = {'datetime': 1.0, 'datetime_error': 1.0, 'int': 0.0, 'int_error': 1.0, 'float': 0.0, 'float_error': 1, 'bool': 0.0, 'bool_error': 1.0} + +BOOL_META = {'bool': .6, 'bool_error': 0.4, + 'datetime': 0.0, 'datetime_error': 1.0, + 'float': 0.6, 'float_error': 0.4, + 'int': 0.6, 'int_error': 0.4} + + +MIXED_NUMERIC_META = {'bool': 0.0, 'bool_error': 1.0, + 'datetime': 0.0, 'datetime_error': 1.0, + 'float': (2/3), 'float_error': (1/3), + 'int': (2/3), 'int_error': (1/3)} + +def assign_values(d, new_values): + d2 = d.copy() + d2.update(new_values) + return d2 + +DATETIME_DTYPE_META = assign_values(ac.default_type_dict, + {'general_type':'datetime', 'exact_type': 'datetime64[ns]'}) + +MIXED_EXACT = assign_values(ac.default_type_dict, {'exact_type': 'Int64', 'general_type': 'int', 'int':1}) + +ONLY_NANS_META = {'datetime': 0, 'datetime_error': 0, 'int': 0, 'int_error': 0, 'float': 0, 'float_error': 0, 'bool': 0, 'bool_error': 0} + +# WEIRD_INT = {'bool': 0.0, 'bool_error': 1.0, +# 'datetime': 0.0, 'datetime_error': 1.0, +# 'float': (2/3), 'float_error': (1/3), +# 'int': (2/3), 'int_error': (1/3)} + + +def test_get_typing_metadata(): + # assert WEIRD_INT == ac.get_typing_metadata(pd.Series([5, 2, 3.1, None, NA])) + assert INT_META == ac.get_typing_metadata(pd.Series(['181', '182', '183', 'a'])) + assert FLOAT_META == ac.get_typing_metadata(pd.Series(['181.1', '182.2', '183', 'a'])) + assert STRING_META == ac.get_typing_metadata(pd.Series(['181.1', 'b', 'c', 'a'])) + assert DATETIME_META == ac.get_typing_metadata(pd.Series(['1981-05-11', '1982-05-11', '1983', 'a'])) + assert DATETIME_EDGECASE_META == ac.get_typing_metadata(pd.Series(['00:01.6', '00:01.6', '00:01.6', None])) + assert DATETIME_DTYPE_META == ac.get_typing_metadata(pd.date_range('2010-01-01', '2010-01-31')) + + assert MIXED_EXACT == ac.get_typing_metadata(pd.Series([NA, 2, 3, NA, NA], dtype='Int64')) + assert MIXED_NUMERIC_META == ac.get_typing_metadata(pd.Series(['a', 2.0, 3.0, None, NA])) + + #there are still problems here, the code isn't properly distinguishing bools from ints and bools + assert BOOL_META == ac.get_typing_metadata(pd.Series(['a', 'b', False, True, False])) + + #only nans + assert ac.get_typing_metadata(pd.Series([None])) == ONLY_NANS_META + assert ac.get_typing_metadata(pd.Series([NA, NA])) == ONLY_NANS_META + assert ac.get_typing_metadata(pd.Series([])) == ONLY_NANS_META + assert ac.get_typing_metadata(pd.Series([nan, NA])) == ONLY_NANS_META + #assert ac.get_typing_metadata(pd.Series([nan, nan])) == ONLY_NANS_META + +def test_recommend_type(): + assert ac.recommend_type(DATETIME_META) == 'datetime' + assert ac.recommend_type(INT_META) == 'int' + assert ac.recommend_type(FLOAT_META) == 'float' + assert ac.recommend_type(STRING_META) == 'string' + assert ac.recommend_type(DATETIME_DTYPE_META) == 'datetime' + + WEIRD_INT_SER = pd.Series(['a', 2, 3, 4, None]) + assert ac.recommend_type( ac.get_typing_metadata(WEIRD_INT_SER)) == 'int' + +def test_smart_to_int(): + assert_series_equal( + ac.smart_to_int(pd.Series(['a', 2, 3, 4, None])), + pd.Series([NA, 2,3,4, NA], dtype='UInt8')) + + + # this should throw an error + # assert_series_equal( + # ac.smart_to_int(pd.Series(['a', 2.0, 3.1, None, NA])), + # pd.Series([NA, 2,3,4, NA], dtype='UInt8')) + + +def test_coerce_series(): + assert_series_equal( + ac.coerce_series(pd.Series(['a', 2, 3, 4, None]), 'int'), + pd.Series([NA, 2,3,4, NA], dtype='UInt8')) + + assert_series_equal( + ac.coerce_series(pd.Series(['a', False, True, None]), 'bool'), + pd.Series([NA, False, True, NA], dtype='boolean')) + + assert_series_equal( + ac.coerce_series(pd.Series(['a', 2.0, 3.0, None, NA]), 'int'), + pd.Series([NA, 2, 3, NA, NA], dtype='UInt8')) + + assert_series_equal( + ac.coerce_series(pd.Series(['a', 2.0, 3.1, None, NA]), 'float'), + pd.Series([nan, 2, 3.1, nan, nan], dtype='float')) + +def atest_autotype_df(): + assert_frame_equal( + ac.auto_type_df( + pd.DataFrame({ + 'int':pd.Series(['a', 2, 3, 4, None]), + 'bool':pd.Series(['a', False, True, None]), + 'int2':pd.Series(['a', 2.0, 3.0, None, NA]), + 'float':pd.Series(['a', 2.0, 3.1, None, NA]), + 'allNA':pd.Series([NA, NA, NA, None, NA]), + 'string_':pd.Series(['a', 'b', 'c', 'd', 'e']) + } + )), + pd.DataFrame({ + 'int' : pd.Series([NA, 2,3,4, NA], dtype='UInt8'), + 'bool': pd.Series([NA, False, True, NA], dtype='boolean'), + 'int2': pd.Series([NA, 2, 3, NA, NA], dtype='UInt8'), + 'float': pd.Series([nan, 2, 3.1, nan, nan], dtype='float'), + 'allNA':pd.Series([NA, NA, NA, None, NA]), + 'string_':pd.Series(['a', 'b', 'c', 'd', 'e']) + })) diff --git a/tests/basic_widget_test.py b/tests/basic_widget_test.py index 4b4e1372..d44c96c0 100644 --- a/tests/basic_widget_test.py +++ b/tests/basic_widget_test.py @@ -3,21 +3,53 @@ from buckaroo.buckaroo_widget import BuckarooWidget +simple_df = pd.DataFrame({'int_col':[1, 2, 3], 'str_col':['a', 'b', 'c']}) + def test_basic_instantiation(): - df = pd.read_csv('./examples/data/2014-01-citibike-tripdata.csv') + df = simple_df w = BuckarooWidget(df) - assert w.dfConfig['totalRows'] == 499 + assert w.dfConfig['totalRows'] == 3 def test_basic_display(): - df = pd.read_csv('./examples/data/2014-01-citibike-tripdata.csv') + df = simple_df w = BuckarooWidget(df) display(w) def test_interpreter(): + #df = pd.read_csv('./examples/data/2014-01-citibike-tripdata.csv') + + w = BuckarooWidget(simple_df) + assert w.operation_results['generated_py_code'] == '''def clean(df): + df['int_col'] = smart_int(df['int_col']) + df['str_col'] = df['str_col'].fillna(value='').astype('string').replace('', None) + return df''' + + temp_ops = w.operations.copy() + temp_ops.append([{"symbol":"dropcol"},{"symbol":"df"},"str_col"]) + w.operations = temp_ops + + tdf = w.operation_results['transformed_df'] + assert w.operation_results['transform_error'] == False + field_names = [ f['name'] for f in tdf['schema']['fields'] ] + assert 'str_col' not in field_names + assert w.operation_results['generated_py_code'] == """def clean(df): + df['int_col'] = smart_int(df['int_col']) + df['str_col'] = df['str_col'].fillna(value='').astype('string').replace('', None) + df.drop('str_col', axis=1, inplace=True) + return df""" + +def atest_symbol_meta(): + """verifies that a symbol with a meta key can be added and + properly interpretted. This should probably be a lower level + parser test + + """ + + df = pd.read_csv('./examples/data/2014-01-citibike-tripdata.csv') w = BuckarooWidget(df) - assert w.operation_results['generated_py_code'] == '#from py widget init' - w.operations = [[{"symbol":"dropcol"},{"symbol":"df"},"starttime"]] + assert w.operation_results['generated_py_code'] == '# instantiation, unused' + w.operations = [[{"symbol":"dropcol", "meta":{}},{"symbol":"df"},"starttime"]] tdf = w.operation_results['transformed_df'] assert w.operation_results['transform_error'] == False @@ -26,12 +58,10 @@ def test_interpreter(): def test_interpreter_errors(): - df = pd.read_csv('./examples/data/2014-01-citibike-tripdata.csv') - w = BuckarooWidget(df) - assert w.operation_results['generated_py_code'] == '#from py widget init' + w = BuckarooWidget(simple_df) w.operations = [ - [{"symbol":"dropcol"},{"symbol":"df"},"starttime"], - [{"symbol":"dropcol"},{"symbol":"df"},"starttime"]] - assert w.operation_results['transform_error'] == '''"['starttime'] not found in axis"''' - + [{"symbol":"dropcol"},{"symbol":"df"},"int_col"], + #dropping the same column will result in an error + [{"symbol":"dropcol"},{"symbol":"df"},"int_col"]] + assert w.operation_results['transform_error'] == '''"['int_col'] not found in axis"''' diff --git a/tests/lispy_test.py b/tests/lispy_test.py new file mode 100644 index 00000000..08db504c --- /dev/null +++ b/tests/lispy_test.py @@ -0,0 +1,29 @@ +import unittest +import pytest +from buckaroo.lisp_utils import split_operations, lists_match + +def test_split_operations(): + + full_ops = [ + [{"symbol":"dropcol", "meta":{"precleaning":True}},{"symbol":"df"},"starttime"], + [{"symbol":"dropcol"},{"symbol":"df"},"starttime"], + ] + + EXPECTED_cleaning_ops = [ + [{"symbol":"dropcol", "meta":{"precleaning":True}},{"symbol":"df"},"starttime"]] + + EXPECTED_user_gen_ops = [ + [{"symbol":"dropcol"},{"symbol":"df"},"starttime"]] + + cleaning_ops, user_gen_ops = split_operations(full_ops) + + assert EXPECTED_cleaning_ops == cleaning_ops + assert EXPECTED_user_gen_ops == user_gen_ops + +def test_lists_match(): + + assert lists_match(["a", "b"], ["a", "b"]) + assert lists_match([["a", "b"]], [["a", "b"]]) + assert not lists_match(["a", "b"], ["a", "b", "c"]) + assert not lists_match([["a", "b"]], [["a", "b", "c"]]) + diff --git a/tsconfig.json b/tsconfig.json index eae169e1..3e9415ee 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,12 +14,17 @@ "skipLibCheck": true, "sourceMap": true, "strict": true, + "experimentalDecorators": true, // This allows us to initialize members in the "initialize" method "strictPropertyInitialization": false, "target": "es2015", "jsx": "react", "baseUrl": "." }, + "typeRoots": [ + "./js/typings", + "./node_modules/@types/" + ], "include": ["js/*.ts", "js/*.tsx", "js/core/*.ts", "js/core/*.tsx", ], "ts-node": { "transpileOnly": true diff --git a/webpack.config.js b/webpack.config.js index 6f90a2c7..11e2acfe 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -95,26 +95,27 @@ const resolve = { module.exports = [ // the following section must be commented out for the nbextension to work // I think it must be enabled for the dev mode of the react app to work - // { - // entry: './examples/index-react18.tsx', - // output: { - // path: path.join(__dirname, '/examples/dist'), - // filename: 'bundle.js' - // }, - // module: { - // rules: demoRules, - // }, - // devtool: 'source-map', - // externals, - // resolve, - // plugins: [new HtmlWebpackPlugin({ - // //template: './examples/index.html' - // template: './examples/index.html' - // })], - // performance - // }, - +/* + { + entry: './examples/index-react18.tsx', + output: { + path: path.join(__dirname, '/examples/dist'), + filename: 'bundle.js' + }, + module: { + rules: demoRules, + }, + devtool: 'source-map', + externals, + resolve, + plugins: [new HtmlWebpackPlugin({ + //template: './examples/index.html' + template: './examples/index.html' + })], + performance + }, +*/ /** * Notebook extension * diff --git a/yarn.lock b/yarn.lock index eeab12e5..58dd8598 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1532,9 +1532,9 @@ url "^0.11.0" "@jupyterlab/builder@^3.0.1": - version "3.6.4" - resolved "https://registry.npmjs.org/@jupyterlab/builder/-/builder-3.6.4.tgz" - integrity sha512-ycTW4P7VRF+67JCb/PlJ358CBsoUYxxkiAEkR6R9QJEY0Zk4S4NuuHq2Sv87Zclpj+xlrxlnSR9WaLq8zEvR7A== + version "3.6.5" + resolved "https://registry.npmjs.org/@jupyterlab/builder/-/builder-3.6.5.tgz" + integrity sha512-J9WO50gxjJ1bUo7BCix0fSxpRySCwaR8AaWNQ5QdYT1ARSGuSnfM9ZyeUOS61DvL3+h7IqsPTQQr5tbhIWv91Q== dependencies: "@lumino/algorithm" "^1.9.0" "@lumino/application" "^1.31.4" @@ -1842,6 +1842,11 @@ mkdirp "^1.0.4" rimraf "^3.0.2" +"@popperjs/core@^2.11.8": + version "2.11.8" + resolved "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz" + integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A== + "@remix-run/router@1.6.2": version "1.6.2" resolved "https://registry.npmjs.org/@remix-run/router/-/router-1.6.2.tgz" @@ -1952,6 +1957,57 @@ dependencies: "@types/node" "*" +"@types/d3-array@^3.0.3": + version "3.0.5" + resolved "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.0.5.tgz" + integrity sha512-Qk7fpJ6qFp+26VeQ47WY0mkwXaiq8+76RJcncDEfMc2ocRzXLO67bLFRNI4OX1aGBoPzsM5Y2T+/m1pldOgD+A== + +"@types/d3-color@*": + version "3.1.0" + resolved "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.0.tgz" + integrity sha512-HKuicPHJuvPgCD+np6Se9MQvS6OCbJmOjGvylzMJRlDwUXjKTTXs6Pwgk79O09Vj/ho3u1ofXnhFOaEWWPrlwA== + +"@types/d3-ease@^3.0.0": + version "3.0.0" + resolved "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.0.tgz" + integrity sha512-aMo4eaAOijJjA6uU+GIeW018dvy9+oH5Y2VPPzjjfxevvGQ/oRDs+tfYC9b50Q4BygRR8yE2QCLsrT0WtAVseA== + +"@types/d3-interpolate@^3.0.1": + version "3.0.1" + resolved "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.1.tgz" + integrity sha512-jx5leotSeac3jr0RePOH1KdR9rISG91QIE4Q2PYTu4OymLTZfA3SrnURSLzKH48HmXVUru50b8nje4E79oQSQw== + dependencies: + "@types/d3-color" "*" + +"@types/d3-path@*": + version "3.0.0" + resolved "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.0.0.tgz" + integrity sha512-0g/A+mZXgFkQxN3HniRDbXMN79K3CdTpLsevj+PXiTcb2hVyvkZUBg37StmgCQkaD84cUJ4uaDAWq7UJOQy2Tg== + +"@types/d3-scale@^4.0.2": + version "4.0.3" + resolved "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.3.tgz" + integrity sha512-PATBiMCpvHJSMtZAMEhc2WyL+hnzarKzI6wAHYjhsonjWJYGq5BXTzQjv4l8m2jO183/4wZ90rKvSeT7o72xNQ== + dependencies: + "@types/d3-time" "*" + +"@types/d3-shape@^3.1.0": + version "3.1.1" + resolved "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.1.tgz" + integrity sha512-6Uh86YFF7LGg4PQkuO2oG6EMBRLuW9cbavUW46zkIO5kuS2PfTqo2o9SkgtQzguBHbLgNnU90UNsITpsX1My+A== + dependencies: + "@types/d3-path" "*" + +"@types/d3-time@*", "@types/d3-time@^3.0.0": + version "3.0.0" + resolved "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz" + integrity sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg== + +"@types/d3-timer@^3.0.0": + version "3.0.0" + resolved "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.0.tgz" + integrity sha512-HNB/9GHqu7Fo8AQiugyJbv6ZxYz58wef0esl4Mv828w1ZKpAshw/uFWVDUcIB9KKFeFKoxS3cHY07FFgtTRZ1g== + "@types/dom4@^2.0.1": version "2.0.2" resolved "https://registry.npmjs.org/@types/dom4/-/dom4-2.0.2.tgz" @@ -2592,7 +2648,17 @@ ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5, ajv@^6.7.0, ajv@^6.9.1: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.0.0, ajv@^8.8.2, ajv@^8.9.0: +ajv@^8.0.0: + version "8.12.0" + resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz" + integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + +ajv@^8.8.2, ajv@^8.9.0: version "8.12.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz" integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== @@ -2994,6 +3060,11 @@ boolbase@^1.0.0: resolved "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz" integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== +bootstrap@^5.1.3: + version "5.3.1" + resolved "https://registry.npmjs.org/bootstrap/-/bootstrap-5.3.1.tgz" + integrity sha512-jzwza3Yagduci2x0rr9MeFSORjcHpt0lRZukZPZQJT1Dth5qzV7XcgGqYzi39KGAVYR8QEDVoO0ubFKOxzMG+g== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" @@ -3194,10 +3265,10 @@ cjs-module-lexer@^1.0.0: resolved "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz" integrity sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA== -classnames@^2.2: - version "2.3.2" - resolved "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz" - integrity sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw== +classnames@^2.2, classnames@^2.2.5: + version "2.2.5" + resolved "https://registry.npmjs.org/classnames/-/classnames-2.2.5.tgz" + integrity sha512-DTt3GhOUDKhh4ONwIJW4lmhyotQmV2LjNlGK/J2hkwUcqcbKkCLAdJPtxQnxnlc7SR3f1CEXCyMmc7WLUsWbNA== clean-css@^4.2.3: version "4.2.4" @@ -3525,6 +3596,11 @@ css-select@^4.1.3: domutils "^2.8.0" nth-check "^2.0.1" +css-unit-converter@^1.1.1: + version "1.1.2" + resolved "https://registry.npmjs.org/css-unit-converter/-/css-unit-converter-1.1.2.tgz" + integrity sha512-IiJwMC8rdZE0+xiEZHeru6YoONC4rfPMqGm2W85jMIbkFvv5nFTwJVFHam2eFrN6txmoUYFAFXiv8ICVeTO0MA== + css-what@^6.0.1: version "6.1.0" resolved "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz" @@ -3550,6 +3626,77 @@ csstype@3.0.10: resolved "https://registry.npmjs.org/csstype/-/csstype-3.0.10.tgz" integrity sha512-2u44ZG2OcNUO9HDp/Jl8C07x6pU/eTR3ncV91SiK3dhG9TWvRVsCoJw14Ckx5DgWkzGA3waZWO3d7pgqpUI/XA== +d3-array@^3.1.6, "d3-array@2 - 3", "d3-array@2.10.0 - 3": + version "3.2.4" + resolved "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz" + integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg== + dependencies: + internmap "1 - 2" + +"d3-color@1 - 3": + version "3.1.0" + resolved "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz" + integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA== + +d3-ease@^3.0.1: + version "3.0.1" + resolved "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz" + integrity sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w== + +"d3-format@1 - 3": + version "3.1.0" + resolved "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz" + integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA== + +d3-interpolate@^3.0.1, "d3-interpolate@1.2.0 - 3": + version "3.0.1" + resolved "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz" + integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g== + dependencies: + d3-color "1 - 3" + +d3-path@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz" + integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ== + +d3-scale@^4.0.2: + version "4.0.2" + resolved "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz" + integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ== + dependencies: + d3-array "2.10.0 - 3" + d3-format "1 - 3" + d3-interpolate "1.2.0 - 3" + d3-time "2.1.1 - 3" + d3-time-format "2 - 4" + +d3-shape@^3.1.0: + version "3.2.0" + resolved "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz" + integrity sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA== + dependencies: + d3-path "^3.1.0" + +"d3-time-format@2 - 4": + version "4.1.0" + resolved "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz" + integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg== + dependencies: + d3-time "1 - 3" + +d3-time@^3.0.0, "d3-time@1 - 3", "d3-time@2.1.1 - 3": + version "3.1.0" + resolved "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz" + integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q== + dependencies: + d3-array "2 - 3" + +d3-timer@^3.0.1: + version "3.0.1" + resolved "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz" + integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA== + data-urls@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz" @@ -3580,6 +3727,11 @@ debug@2.6.9: dependencies: ms "2.0.0" +decimal.js-light@^2.4.1: + version "2.5.1" + resolved "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz" + integrity sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg== + dedent@^0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz" @@ -4094,7 +4246,7 @@ etag@~1.8.1: resolved "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -eventemitter3@^4.0.0: +eventemitter3@^4.0.0, eventemitter3@^4.0.1: version "4.0.7" resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== @@ -4197,6 +4349,11 @@ fast-diff@^1.1.2: resolved "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz" integrity sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw== +fast-equals@^5.0.0: + version "5.0.1" + resolved "https://registry.npmjs.org/fast-equals/-/fast-equals-5.0.1.tgz" + integrity sha512-WF1Wi8PwwSY7/6Kx0vKXtw8RwuSGoM1bvDaJbu7MxDlR1vovZjIAKrnzyrThgAjm6JDTu0fVgWXDlMGspodfoQ== + fast-glob@^3.2.9: version "3.2.12" resolved "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz" @@ -4905,6 +5062,11 @@ internal-slot@^1.0.3, internal-slot@^1.0.5: has "^1.0.3" side-channel "^1.0.4" +"internmap@1 - 2": + version "2.0.3" + resolved "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz" + integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== + interpret@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz" @@ -5866,7 +6028,7 @@ lodash.merge@^4.6.2: resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.7.0: +lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.7.0: version "4.17.21" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -6876,6 +7038,11 @@ postcss-selector-parser@^6.0.10, postcss-selector-parser@^6.0.11, postcss-select cssesc "^3.0.0" util-deprecate "^1.0.2" +postcss-value-parser@^3.3.0: + version "3.3.1" + resolved "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz" + integrity sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ== + postcss-value-parser@^4.1.0, postcss-value-parser@^4.2.0: version "4.2.0" resolved "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz" @@ -6958,7 +7125,7 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" -prop-types@^15.0.0, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: +prop-types@^15.0.0, prop-types@^15.6.0, prop-types@^15.6.1, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: version "15.8.1" resolved "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz" integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== @@ -7050,7 +7217,15 @@ raw-loader@~4.0.0: loader-utils "^2.0.0" schema-utils "^3.0.0" -"react-dom@^15.3.0 || 16 || 17", react-dom@^17.0.1, react-dom@>=15.0.0: +"react-dom@^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0", "react-dom@^16.0.0 || ^17.0.0 || ^18.0.0", "react-dom@^16.3.0 || ^17.0.0 || ^18.0.0", react-dom@^18.0.0, react-dom@>=15.0.0: + version "18.2.0" + resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz" + integrity sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g== + dependencies: + loose-envify "^1.1.0" + scheduler "^0.23.0" + +"react-dom@^15.3.0 || 16 || 17", react-dom@^17.0.1: version "17.0.2" resolved "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz" integrity sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA== @@ -7059,15 +7234,7 @@ raw-loader@~4.0.0: object-assign "^4.1.1" scheduler "^0.20.2" -"react-dom@^16.3.0 || ^17.0.0 || ^18.0.0", react-dom@^18.0.0: - version "18.2.0" - resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz" - integrity sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g== - dependencies: - loose-envify "^1.1.0" - scheduler "^0.23.0" - -react-is@^16.13.1, react-is@^16.6.0, react-is@^16.7.0, react-is@^16.9.0: +react-is@^16.10.2, react-is@^16.13.1, react-is@^16.6.0, react-is@^16.7.0, react-is@^16.9.0: version "16.13.1" resolved "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== @@ -7095,6 +7262,13 @@ react-popper@^1.3.7: typed-styles "^0.0.7" warning "^4.0.2" +react-resize-detector@^8.0.4: + version "8.1.0" + resolved "https://registry.npmjs.org/react-resize-detector/-/react-resize-detector-8.1.0.tgz" + integrity sha512-S7szxlaIuiy5UqLhLL1KY3aoyGHbZzsTpYal9eYMwCyKqoqoVLCmIgAgNyIM1FhnP2KyBygASJxdhejrzjMb+w== + dependencies: + lodash "^4.17.21" + react-router-dom@^5.2.0: version "5.3.4" resolved "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.3.4.tgz" @@ -7130,7 +7304,15 @@ react-router@5.3.4: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" -react-transition-group@^2.9.0: +react-smooth@^2.0.2, react-smooth@^2.0.3: + version "2.0.3" + resolved "https://registry.npmjs.org/react-smooth/-/react-smooth-2.0.3.tgz" + integrity sha512-yl4y3XiMorss7ayF5QnBiSprig0+qFHui8uh7Hgg46QX5O+aRMRKlfGGNGLHno35JkQSvSYY8eCWkBfHfrSHfg== + dependencies: + fast-equals "^5.0.0" + react-transition-group "2.9.0" + +react-transition-group@^2.9.0, react-transition-group@2.9.0: version "2.9.0" resolved "https://registry.npmjs.org/react-transition-group/-/react-transition-group-2.9.0.tgz" integrity sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg== @@ -7140,7 +7322,14 @@ react-transition-group@^2.9.0: prop-types "^15.6.2" react-lifecycles-compat "^3.0.4" -"react@^15.3.0 || 16 || 17", react@^17.0.1, react@>=0.14.0, react@>=15.0.0, react@>=16, "react@0.14.x || ^15.0.0 || ^16.0.0 || ^17.0.0", react@17.0.2: +"react@^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0", "react@^16.0.0 || ^17.0.0 || ^18.0.0", "react@^16.3.0 || ^17.0.0 || ^18.0.0", react@^18.0.0, react@^18.2.0, react@>=15, react@>=15.0.0, react@>=16.8: + version "18.2.0" + resolved "https://registry.npmjs.org/react/-/react-18.2.0.tgz" + integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ== + dependencies: + loose-envify "^1.1.0" + +"react@^15.3.0 || 16 || 17", react@^17.0.1, react@>=0.14.0, react@>=16, "react@0.14.x || ^15.0.0 || ^16.0.0 || ^17.0.0", react@17.0.2: version "17.0.2" resolved "https://registry.npmjs.org/react/-/react-17.0.2.tgz" integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== @@ -7148,13 +7337,6 @@ react-transition-group@^2.9.0: loose-envify "^1.1.0" object-assign "^4.1.1" -"react@^16.3.0 || ^17.0.0 || ^18.0.0", react@^18.0.0, react@^18.2.0, react@>=15, react@>=16.8: - version "18.2.0" - resolved "https://registry.npmjs.org/react/-/react-18.2.0.tgz" - integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ== - dependencies: - loose-envify "^1.1.0" - read-pkg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz" @@ -7193,6 +7375,28 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +recharts-scale@^0.4.4: + version "0.4.5" + resolved "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz" + integrity sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w== + dependencies: + decimal.js-light "^2.4.1" + +recharts@^2.7.3: + version "2.7.3" + resolved "https://registry.npmjs.org/recharts/-/recharts-2.7.3.tgz" + integrity sha512-cKoO9jUZRQavn06H6Ih2EcG82zUNdQH3OEGWVCmluSDyp3d7fIpDAsbMTd8hE8+T+MD8P76iicv/J4pJspDP7A== + dependencies: + classnames "^2.2.5" + eventemitter3 "^4.0.1" + lodash "^4.17.19" + react-is "^16.10.2" + react-resize-detector "^8.0.4" + react-smooth "^2.0.2" + recharts-scale "^0.4.4" + reduce-css-calc "^2.1.8" + victory-vendor "^36.6.8" + rechoir@^0.7.0: version "0.7.1" resolved "https://registry.npmjs.org/rechoir/-/rechoir-0.7.1.tgz" @@ -7200,6 +7404,14 @@ rechoir@^0.7.0: dependencies: resolve "^1.9.0" +reduce-css-calc@^2.1.8: + version "2.1.8" + resolved "https://registry.npmjs.org/reduce-css-calc/-/reduce-css-calc-2.1.8.tgz" + integrity sha512-8liAVezDmUcH+tdzoEGrhfbGcP7nOV4NkGE3a74+qqvE7nt9i4sKLGBuZNOnpI4WiGksiNPklZxva80061QiPg== + dependencies: + css-unit-converter "^1.1.1" + postcss-value-parser "^3.3.0" + regenerate-unicode-properties@^10.1.0: version "10.1.0" resolved "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.1.0.tgz" @@ -7537,9 +7749,9 @@ semver@^7.3.4: lru-cache "^6.0.0" semver@^7.3.5: - version "7.5.1" - resolved "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz" - integrity sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw== + version "7.5.4" + resolved "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== dependencies: lru-cache "^6.0.0" @@ -8527,6 +8739,26 @@ vary@~1.1.2: resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== +victory-vendor@^36.6.8: + version "36.6.11" + resolved "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.6.11.tgz" + integrity sha512-nT8kCiJp8dQh8g991J/R5w5eE2KnO8EAIP0xocWlh9l2okngMWglOPoMZzJvek8Q1KUc4XE/mJxTZnvOB1sTYg== + dependencies: + "@types/d3-array" "^3.0.3" + "@types/d3-ease" "^3.0.0" + "@types/d3-interpolate" "^3.0.1" + "@types/d3-scale" "^4.0.2" + "@types/d3-shape" "^3.1.0" + "@types/d3-time" "^3.0.0" + "@types/d3-timer" "^3.0.0" + d3-array "^3.1.6" + d3-ease "^3.0.1" + d3-interpolate "^3.0.1" + d3-scale "^4.0.2" + d3-shape "^3.1.0" + d3-time "^3.0.0" + d3-timer "^3.0.1" + walker@^1.0.8: version "1.0.8" resolved "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz"